diff --git a/models.yaml b/models.yaml index 86811707..dcf71852 100644 --- a/models.yaml +++ b/models.yaml @@ -72,17 +72,29 @@ models: chat_model: models/gemini-1.5-flash-latest llm_hass_api: assist + - model_id: claude-3-5-sonnet domain: anthropic - description: Google Generative AI integration using gemini flash (v1.5) + description: Anthropic integration using Claude 3.5 Sonnet urls: - - https://blog.google/products/gemini/google-gemini-new-features-july-2024/ + - https://www.anthropic.com/news/claude-3-5-sonnet config_entry_data: api_key: !secret anthropic_api_key config_entry_options: chat_model: claude-3-5-sonnet-20240620 llm_hass_api: assist + - model_id: claude-3-haiku + domain: anthropic + description: Anthropic integration using Claude 3 Haiku + urls: + - https://www.anthropic.com/news/claude-3-haiku + config_entry_data: + api_key: !secret anthropic_api_key + config_entry_options: + chat_model: claude-3-haiku-20240307 + llm_hass_api: assist + # # Open Source smaller function calling models # diff --git a/reports/README.md b/reports/README.md index 6b46593b..50c44422 100644 --- a/reports/README.md +++ b/reports/README.md @@ -15,6 +15,7 @@ | mistral-v3 | 3.8% (CI: 4.2%, 2024.8.0b) | 2.0% (CI: 4.0%, 2024.8.0dev) | 10.3% (CI: 4.6%, 2024.8.0b) | | xlam-1b | | 27.1% (CI: 12.6%, 2024.8.0b0) | | | claude-3-5-sonnet | | 95.9% (CI: 5.5%, 2024.9.0b2) | | +| claude-3-haiku | | 98.0% (CI: 4.0%, 2024.9.0b2) | | Implementation notes: - CI is large given small number of samples in the datasets. @@ -96,26 +97,27 @@ config: themeVariables: xyChart: titleColor: "#ff0000" - plotColorPalette: "#d5a6bd, #0f9d58, #4285f4, #4285f4, #ff6d01, #674ea7, #6aa84f, #34a853, #46bdc6, #fbbc04, #1155cc, #d9ead3, #4285f4" + plotColorPalette: "#d5a6bd, #0f9d58, #4285f4, #4285f4, #0f9d58, #ff6d01, #674ea7, #6aa84f, #34a853, #46bdc6, #fbbc04, #1155cc, #d9ead3, #4285f4" --- xychart-beta title "assist-mini" - x-axis "Model" [assistant, gpt-4o-mini, gemini-1.5-flash, claude-3-5-sonnet, functionary-small-v2.5, mistral-v3, llama3-groq-tool-use, llama3.1, xlam-7b, assist-llm, home-llm, xlam-1b, .] + x-axis "Model" [assistant, gpt-4o-mini, gemini-1.5-flash, claude-3-5-sonnet, claude-3-haiku, functionary-small-v2.5, mistral-v3, llama3-groq-tool-use, llama3.1, xlam-7b, assist-llm, home-llm, xlam-1b, .] y-axis "Score" 1 --> 100 - bar [63.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - bar [0.0, 98.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - bar [0.0, 0.0, 98.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - bar [0.0, 0.0, 0.0, 95.9, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - bar [0.0, 0.0, 0.0, 0.0, 63.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - bar [0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 51.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 83.7, 0.0, 0.0, 0.0, 0.0, 0.0] - bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 85.7, 0.0, 0.0, 0.0, 0.0] - bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 81.6, 0.0, 0.0, 0.0] - bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 34.7, 0.0, 0.0] - bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 27.1, 0.0] - bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0] + bar [63.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + bar [0.0, 98.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + bar [0.0, 0.0, 98.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + bar [0.0, 0.0, 0.0, 95.9, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + bar [0.0, 0.0, 0.0, 0.0, 98.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + bar [0.0, 0.0, 0.0, 0.0, 0.0, 63.3, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 51.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 83.7, 0.0, 0.0, 0.0, 0.0, 0.0] + bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 85.7, 0.0, 0.0, 0.0, 0.0] + bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 81.6, 0.0, 0.0, 0.0] + bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 34.7, 0.0, 0.0] + bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 27.1, 0.0] + bar [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0] ``` ### intents @@ -207,10 +209,18 @@ More information: ### claude-3-5-sonnet -Google Generative AI integration using gemini flash (v1.5) +Anthropic integration using Claude 3.5 Sonnet More information: -- https://blog.google/products/gemini/google-gemini-new-features-july-2024/ +- https://www.anthropic.com/news/claude-3-5-sonnet + + +### claude-3-haiku + +Anthropic integration using Claude 3 Haiku + +More information: +- https://www.anthropic.com/news/claude-3-haiku ### functionary-small-v2.5 diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-dining_room_light_off.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-dining_room_light_off.yaml new file mode 100644 index 00000000..a688fcef --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-dining_room_light_off.yaml @@ -0,0 +1,79 @@ +--- +uuid: 31c8000e-04a4-478b-9d49-d40a11194013 +task_id: dom1_pl_lights_lights-dining_room_light_off +category: light +task: + input_text: Dining room light off + expect_changes: + light.dining_room_light: + state: 'off' + attributes: + brightness: null + color_mode: null +response: I have turned off the Dining Room Light. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Dining room light off + context: + id: 01J6SPSMEQDS0JXW3RY77YE9AM + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:48:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Kitchen Light\n domain: light\n state: 'on'\n areas:\ + \ Kitchen\n attributes:\n brightness: '100'\n- names: Living Room Light\n\ + \ domain: light\n state: 'off'\n areas: Living Room\n attributes:\n \ + \ brightness:\n- names: Dining Room Light\n domain: light\n state: 'on'\n\ + \ areas: Dining Room\n attributes:\n brightness: '100'\n- names: Bedroom\ + \ 1 Light\n domain: light\n state: 'off'\n areas: Bedroom 1\n- names: Bedroom\ + \ 2 Light\n domain: light\n state: 'off'\n areas: Bedroom 2\n- names: Bedroom\ + \ 3 Light\n domain: light\n state: 'off'\n areas: Bedroom 3\n- names: Bedroom\ + \ 4 Light\n domain: light\n state: 'off'\n areas: Bedroom 4\n- names: Garden\ + \ Light\n domain: light\n state: 'off'\n areas: Backyard\n attributes:\n\ + \ brightness:\n" + messages: + - role: user + content: Dining room light off + - role: assistant + content: + - type: tool_use + id: toolu_01TFiYNB4X2JxFnjNt6vTF1K + name: HassTurnOff + input: + domain: + - light + name: Dining Room Light + - role: user + content: + - type: tool_result + tool_use_id: toolu_01TFiYNB4X2JxFnjNt6vTF1K + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Dining Room Light", "type": "entity", "id": + "light.dining_room_light"}], "failed": []}}' + - role: assistant + content: + - type: text + text: I have turned off the Dining Room Light. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + domain: + - light + name: Dining Room Light diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-kitchen_light_off.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-kitchen_light_off.yaml new file mode 100644 index 00000000..1bbd0218 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-kitchen_light_off.yaml @@ -0,0 +1,77 @@ +--- +uuid: 1edf6335-7b44-4490-a082-57cf359b6bd0 +task_id: dom1_pl_lights_lights-kitchen_light_off +category: light +task: + input_text: Kitchen light off + expect_changes: + light.kitchen_light: + state: 'off' + attributes: + brightness: null + color_mode: null +response: The kitchen light has been turned off. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Kitchen light off + context: + id: 01J6SQ7DNJ8EGRJ17DB0T8ASJE + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:56:30. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Kitchen Light\n domain: light\n state: 'on'\n areas:\ + \ Kitchen\n attributes:\n brightness: '100'\n- names: Living Room Light\n\ + \ domain: light\n state: 'off'\n areas: Living Room\n attributes:\n \ + \ brightness:\n- names: Dining Room Light\n domain: light\n state: 'on'\n\ + \ areas: Dining Room\n attributes:\n brightness: '100'\n- names: Bedroom\ + \ 1 Light\n domain: light\n state: 'off'\n areas: Bedroom 1\n- names: Bedroom\ + \ 2 Light\n domain: light\n state: 'off'\n areas: Bedroom 2\n- names: Bedroom\ + \ 3 Light\n domain: light\n state: 'off'\n areas: Bedroom 3\n- names: Bedroom\ + \ 4 Light\n domain: light\n state: 'off'\n areas: Bedroom 4\n- names: Garden\ + \ Light\n domain: light\n state: 'off'\n areas: Backyard\n attributes:\n\ + \ brightness:\n" + messages: + - role: user + content: Kitchen light off + - role: assistant + content: + - type: tool_use + id: toolu_01AvSVpEZrT7bHJRCPN1KaMi + name: HassTurnOff + input: + name: Kitchen Light + domain: light + - role: user + content: + - type: tool_result + tool_use_id: toolu_01AvSVpEZrT7bHJRCPN1KaMi + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Kitchen Light", "type": "entity", "id": "light.kitchen_light"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The kitchen light has been turned off. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + name: Kitchen Light + domain: light diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-please_turn_on_the_kitchen_light.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-please_turn_on_the_kitchen_light.yaml new file mode 100644 index 00000000..a9c67785 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-please_turn_on_the_kitchen_light.yaml @@ -0,0 +1,77 @@ +--- +uuid: 1cdc6296-2d9b-40af-89d6-647cbd598d16 +task_id: dom1_pl_lights_lights-please_turn_on_the_kitchen_light +category: light +task: + input_text: Please turn on the kitchen light + expect_changes: + light.kitchen_light: + state: 'on' + attributes: + brightness: 0 + color_mode: brightness +response: I've turned on the Kitchen Light. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Please turn on the kitchen light + context: + id: 01J6SPNZ2A72M83JJE2EB5KTZC + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:46:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Kitchen Light\n domain: light\n state: 'off'\n \ + \ areas: Kitchen\n attributes:\n brightness:\n- names: Living Room Light\n\ + \ domain: light\n state: 'on'\n areas: Living Room\n attributes:\n \ + \ brightness: '100'\n- names: Dining Room Light\n domain: light\n state:\ + \ 'on'\n areas: Dining Room\n attributes:\n brightness: '100'\n- names:\ + \ Bedroom 1 Light\n domain: light\n state: 'off'\n areas: Bedroom 1\n-\ + \ names: Bedroom 2 Light\n domain: light\n state: 'off'\n areas: Bedroom\ + \ 2\n- names: Bedroom 3 Light\n domain: light\n state: 'off'\n areas: Bedroom\ + \ 3\n- names: Bedroom 4 Light\n domain: light\n state: 'off'\n areas: Bedroom\ + \ 4\n- names: Garden Light\n domain: light\n state: 'on'\n areas: Backyard\n\ + \ attributes:\n brightness: '100'\n" + messages: + - role: user + content: Please turn on the kitchen light + - role: assistant + content: + - type: tool_use + id: toolu_01JTjfrJj5BqAndXa6Jaxtp4 + name: HassTurnOn + input: + name: Kitchen Light + domain: light + - role: user + content: + - type: tool_result + tool_use_id: toolu_01JTjfrJj5BqAndXa6Jaxtp4 + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Kitchen Light", "type": "entity", "id": "light.kitchen_light"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I've turned on the Kitchen Light. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Kitchen Light + domain: light diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-turn_on_the_backyard_light.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-turn_on_the_backyard_light.yaml new file mode 100644 index 00000000..5c920fe6 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-turn_on_the_backyard_light.yaml @@ -0,0 +1,82 @@ +--- +uuid: d02652cf-c7f5-4bfb-ae96-36cabbe01cf3 +task_id: dom1_pl_lights_lights-turn_on_the_backyard_light +category: light +task: + input_text: Turn on the Backyard light + expect_changes: + light.garden_light: + state: 'on' + attributes: + brightness: 0 + color_mode: brightness +response: Backyard light has been turned on. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Turn on the Backyard light + context: + id: 01J6SPQSE23D009NJJ88TA5G4B + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:47:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Kitchen Light\n domain: light\n state: 'off'\n \ + \ areas: Kitchen\n attributes:\n brightness:\n- names: Living Room Light\n\ + \ domain: light\n state: 'off'\n areas: Living Room\n attributes:\n \ + \ brightness:\n- names: Dining Room Light\n domain: light\n state: 'on'\n\ + \ areas: Dining Room\n attributes:\n brightness: '100'\n- names: Bedroom\ + \ 1 Light\n domain: light\n state: 'off'\n areas: Bedroom 1\n- names: Bedroom\ + \ 2 Light\n domain: light\n state: 'off'\n areas: Bedroom 2\n- names: Bedroom\ + \ 3 Light\n domain: light\n state: 'off'\n areas: Bedroom 3\n- names: Bedroom\ + \ 4 Light\n domain: light\n state: 'off'\n areas: Bedroom 4\n- names: Garden\ + \ Light\n domain: light\n state: 'off'\n areas: Backyard\n attributes:\n\ + \ brightness:\n" + messages: + - role: user + content: Turn on the Backyard light + - role: assistant + content: + - type: tool_use + id: toolu_01S8udAmWx9Wm43KEc4oHZ77 + name: HassTurnOn + input: + name: Garden Light + domain: + - light + area: Backyard + - role: user + content: + - type: tool_result + tool_use_id: toolu_01S8udAmWx9Wm43KEc4oHZ77 + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Backyard", "type": "area", "id": "backyard"}, + {"name": "Garden Light", "type": "entity", "id": "light.garden_light"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: Backyard light has been turned on. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Garden Light + domain: + - light + area: Backyard diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-turn_on_the_living_room_light.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-turn_on_the_living_room_light.yaml new file mode 100644 index 00000000..73505795 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_lights_lights-turn_on_the_living_room_light.yaml @@ -0,0 +1,77 @@ +--- +uuid: cabea44a-2460-4286-a5ec-35cb511b5cb0 +task_id: dom1_pl_lights_lights-turn_on_the_living_room_light +category: light +task: + input_text: Turn on the living room light + expect_changes: + light.living_room_light: + state: 'on' + attributes: + brightness: 0 + color_mode: brightness +response: Turning on the living room light. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Turn on the living room light + context: + id: 01J6SRPQEP8EGRJ17DB0T8ASJE + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:22:20. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Kitchen Light\n domain: light\n state: 'off'\n \ + \ areas: Kitchen\n attributes:\n brightness:\n- names: Living Room Light\n\ + \ domain: light\n state: 'off'\n areas: Living Room\n attributes:\n \ + \ brightness:\n- names: Dining Room Light\n domain: light\n state: 'on'\n\ + \ areas: Dining Room\n attributes:\n brightness: '100'\n- names: Bedroom\ + \ 1 Light\n domain: light\n state: 'off'\n areas: Bedroom 1\n- names: Bedroom\ + \ 2 Light\n domain: light\n state: 'off'\n areas: Bedroom 2\n- names: Bedroom\ + \ 3 Light\n domain: light\n state: 'off'\n areas: Bedroom 3\n- names: Bedroom\ + \ 4 Light\n domain: light\n state: 'off'\n areas: Bedroom 4\n- names: Garden\ + \ Light\n domain: light\n state: 'on'\n areas: Backyard\n attributes:\n\ + \ brightness: '100'\n" + messages: + - role: user + content: Turn on the living room light + - role: assistant + content: + - type: tool_use + id: toolu_01QLkwrRkcsj5jn6TtafCE7G + name: HassTurnOn + input: + name: Living Room Light + domain: light + - role: user + content: + - type: tool_result + tool_use_id: toolu_01QLkwrRkcsj5jn6TtafCE7G + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room Light", "type": "entity", "id": + "light.living_room_light"}], "failed": []}}' + - role: assistant + content: + - type: text + text: Turning on the living room light. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Living Room Light + domain: light diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-add_clean_the_kitchen_to_my_todo_list.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-add_clean_the_kitchen_to_my_todo_list.yaml new file mode 100644 index 00000000..1752e620 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-add_clean_the_kitchen_to_my_todo_list.yaml @@ -0,0 +1,64 @@ +--- +uuid: c670faa3-7e48-4dfb-9e89-d847927672a9 +task_id: dom1_pl_todo_todo-add_clean_the_kitchen_to_my_todo_list +category: todo +task: + input_text: Add clean the kitchen to my todo list + expect_changes: + todo.personal_tasks: + state: '1' + attributes: null +response: I've added "clean the kitchen" to your Personal Tasks todo list. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Add clean the kitchen to my todo list + context: + id: 01J6SPX9T10NPVNK65P113755G + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:50:59. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Personal Tasks\n domain: todo\n state: '0'\n" + messages: + - role: user + content: Add clean the kitchen to my todo list + - role: assistant + content: + - type: tool_use + id: toolu_019BPmy4BAbfnwkdpKf77DAD + name: HassListAddItem + input: + item: clean the kitchen + name: Personal Tasks + - role: user + content: + - type: tool_result + tool_use_id: toolu_019BPmy4BAbfnwkdpKf77DAD + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [], "failed": []}}' + - role: assistant + content: + - type: text + text: I've added "clean the kitchen" to your Personal Tasks todo list. + - event_type: tool_call + data: + tool_name: HassListAddItem + tool_args: + item: clean the kitchen + name: Personal Tasks diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-add_history_homework_to_my_personal_tasks.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-add_history_homework_to_my_personal_tasks.yaml new file mode 100644 index 00000000..37ba45c4 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-add_history_homework_to_my_personal_tasks.yaml @@ -0,0 +1,67 @@ +--- +uuid: 828941bd-af23-419e-9c44-c391c84dfe2d +task_id: dom1_pl_todo_todo-add_history_homework_to_my_personal_tasks +category: todo +task: + input_text: Add history homework to my personal tasks + expect_changes: + todo.personal_tasks: + state: '1' + attributes: null +response: The item "history homework" has been added to your Personal Tasks list. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Add history homework to my personal tasks + context: + id: 01J6SPVEYB4TGCQC59R8FJJCX0 + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:49:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Personal Tasks\n domain: todo\n state: '0'\n" + messages: + - role: user + content: Add history homework to my personal tasks + - role: assistant + content: + - type: text + text: 'Okay, let''s add "history homework" to your personal tasks list:' + - type: tool_use + id: toolu_014DkZmjpDsf7TEBedpkPjqf + name: HassListAddItem + input: + name: Personal Tasks + item: history homework + - role: user + content: + - type: tool_result + tool_use_id: toolu_014DkZmjpDsf7TEBedpkPjqf + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [], "failed": []}}' + - role: assistant + content: + - type: text + text: The item "history homework" has been added to your Personal Tasks + list. + - event_type: tool_call + data: + tool_name: HassListAddItem + tool_args: + name: Personal Tasks + item: history homework diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-add_history_homework_to_my_tasks.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-add_history_homework_to_my_tasks.yaml new file mode 100644 index 00000000..8f7b945c --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-add_history_homework_to_my_tasks.yaml @@ -0,0 +1,66 @@ +--- +uuid: 5f95624d-84e7-4922-8087-576adcdafb15 +task_id: dom1_pl_todo_todo-add_history_homework_to_my_tasks +category: todo +task: + input_text: Add history homework to my tasks + expect_changes: + todo.personal_tasks: + state: '1' + attributes: null +response: I've added "history homework" to your Personal Tasks to-do list. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Add history homework to my tasks + context: + id: 01J6SQ8XQWEVC7WJHENTWJ76BX + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:57:20. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Personal Tasks\n domain: todo\n state: '0'\n" + messages: + - role: user + content: Add history homework to my tasks + - role: assistant + content: + - type: text + text: 'Okay, let''s add "history homework" to your Personal Tasks list:' + - type: tool_use + id: toolu_017ySDfbFMVMVpdXuuJakuYM + name: HassListAddItem + input: + name: Personal Tasks + item: history homework + - role: user + content: + - type: tool_result + tool_use_id: toolu_017ySDfbFMVMVpdXuuJakuYM + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [], "failed": []}}' + - role: assistant + content: + - type: text + text: I've added "history homework" to your Personal Tasks to-do list. + - event_type: tool_call + data: + tool_name: HassListAddItem + tool_args: + name: Personal Tasks + item: history homework diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-put_history_homework_on_personal_tasks_todo_list.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-put_history_homework_on_personal_tasks_todo_list.yaml new file mode 100644 index 00000000..978cce98 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/dom1_pl_todo_todo-put_history_homework_on_personal_tasks_todo_list.yaml @@ -0,0 +1,66 @@ +--- +uuid: a8235d46-6153-47c6-8f8d-af7fbc877cd7 +task_id: dom1_pl_todo_todo-put_history_homework_on_personal_tasks_todo_list +category: todo +task: + input_text: Put history homework on personal tasks todo list + expect_changes: + todo.personal_tasks: + state: '1' + attributes: null +response: History homework has been added to your personal tasks todo list. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Put history homework on personal tasks todo list + context: + id: 01J6SR8KS5EVC7WJHENTWJ76BX + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:14:38. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Personal Tasks\n domain: todo\n state: '0'\n" + messages: + - role: user + content: Put history homework on personal tasks todo list + - role: assistant + content: + - type: text + text: Okay, let's add "history homework" to your personal tasks todo list. + - type: tool_use + id: toolu_015sC2PXhN4iNNaj21yTZqv1 + name: HassListAddItem + input: + name: Personal Tasks + item: history homework + - role: user + content: + - type: tool_result + tool_use_id: toolu_015sC2PXhN4iNNaj21yTZqv1 + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [], "failed": []}}' + - role: assistant + content: + - type: text + text: History homework has been added to your personal tasks todo list. + - event_type: tool_call + data: + tool_name: HassListAddItem + tool_args: + name: Personal Tasks + item: history homework diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-close_the_garage_door.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-close_the_garage_door.yaml new file mode 100644 index 00000000..aa21c2e1 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-close_the_garage_door.yaml @@ -0,0 +1,71 @@ +--- +uuid: 5b5f400a-8e53-428a-a5b4-c6b8ff85d2ff +task_id: home1_us_cover_garage_cover_garage-close_the_garage_door +category: cover +task: + input_text: Close the garage door + expect_changes: + cover.garage_door_opener: + state: closed + attributes: null +response: The garage door has been closed. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Close the garage door + context: + id: 01J6SPAJ8MHAXA9CGH15W2AQ2G + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:40:45. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Garage Door Opener\n domain: cover\n state: open\n\ + \ areas: Garage\n attributes:\n current_position: '100'\n device_class:\ + \ garage\n- names: Garage Door Opener\n domain: light\n state: 'off'\n \ + \ areas: Garage\n" + messages: + - role: user + content: Close the garage door + - role: assistant + content: + - type: tool_use + id: toolu_011TPA4eoHasK7UYg8rHRhsp + name: HassTurnOff + input: + domain: cover + name: Garage Door Opener + area: Garage + - role: user + content: + - type: tool_result + tool_use_id: toolu_011TPA4eoHasK7UYg8rHRhsp + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Garage", "type": "area", "id": "garage"}, {"name": + "Garage Door Opener", "type": "entity", "id": "cover.garage_door_opener"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The garage door has been closed. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + domain: cover + name: Garage Door Opener + area: Garage diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-open_the_garage_door.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-open_the_garage_door.yaml new file mode 100644 index 00000000..7f153cdc --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-open_the_garage_door.yaml @@ -0,0 +1,71 @@ +--- +uuid: f159290a-b484-486f-8e7d-91e751cd0a19 +task_id: home1_us_cover_garage_cover_garage-open_the_garage_door +category: cover +task: + input_text: Open the garage door + expect_changes: + cover.garage_door_opener: + state: open + attributes: null +response: The garage door has been opened. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Open the garage door + context: + id: 01J6SQACSC4R91Q9AW95A1EV7E + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:58:08. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Garage Door Opener\n domain: cover\n state: closed\n\ + \ areas: Garage\n attributes:\n current_position: '0'\n device_class:\ + \ garage\n- names: Garage Door Opener\n domain: light\n state: 'off'\n \ + \ areas: Garage\n" + messages: + - role: user + content: Open the garage door + - role: assistant + content: + - type: tool_use + id: toolu_01FsvqXS281zsN1hT7uzXveR + name: HassTurnOn + input: + domain: cover + name: Garage Door Opener + area: Garage + - role: user + content: + - type: tool_result + tool_use_id: toolu_01FsvqXS281zsN1hT7uzXveR + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Garage", "type": "area", "id": "garage"}, {"name": + "Garage Door Opener", "type": "entity", "id": "cover.garage_door_opener"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The garage door has been opened. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + domain: cover + name: Garage Door Opener + area: Garage diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-please_close_the_garage_door.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-please_close_the_garage_door.yaml new file mode 100644 index 00000000..ded992a0 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-please_close_the_garage_door.yaml @@ -0,0 +1,71 @@ +--- +uuid: 76ae3c09-d1a5-4edd-9c2b-0da944412ad6 +task_id: home1_us_cover_garage_cover_garage-please_close_the_garage_door +category: cover +task: + input_text: Please close the garage door + expect_changes: + cover.garage_door_opener: + state: closed + attributes: null +response: Closing the garage door. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Please close the garage door + context: + id: 01J6SR0R2M4R91Q9AW95A1EV7E + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:10:20. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Garage Door Opener\n domain: cover\n state: open\n\ + \ areas: Garage\n attributes:\n current_position: '100'\n device_class:\ + \ garage\n- names: Garage Door Opener\n domain: light\n state: 'off'\n \ + \ areas: Garage\n" + messages: + - role: user + content: Please close the garage door + - role: assistant + content: + - type: tool_use + id: toolu_0128DXtiE1fpyFFckTNAYy6b + name: HassTurnOff + input: + domain: cover + name: Garage Door Opener + area: Garage + - role: user + content: + - type: tool_result + tool_use_id: toolu_0128DXtiE1fpyFFckTNAYy6b + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Garage", "type": "area", "id": "garage"}, {"name": + "Garage Door Opener", "type": "entity", "id": "cover.garage_door_opener"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: Closing the garage door. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + domain: cover + name: Garage Door Opener + area: Garage diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-please_open_the_garage_door.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-please_open_the_garage_door.yaml new file mode 100644 index 00000000..a695416f --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_cover_garage_cover_garage-please_open_the_garage_door.yaml @@ -0,0 +1,71 @@ +--- +uuid: 7ab58661-3334-4b0c-ba2f-eb11fc1b5e71 +task_id: home1_us_cover_garage_cover_garage-please_open_the_garage_door +category: cover +task: + input_text: Please open the garage door + expect_changes: + cover.garage_door_opener: + state: open + attributes: null +response: The garage door has been opened. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Please open the garage door + context: + id: 01J6SPCXBFYWEE3YM9M237NC59 + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:42:02. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Garage Door Opener\n domain: cover\n state: closed\n\ + \ areas: Garage\n attributes:\n current_position: '0'\n device_class:\ + \ garage\n- names: Garage Door Opener\n domain: light\n state: 'off'\n \ + \ areas: Garage\n" + messages: + - role: user + content: Please open the garage door + - role: assistant + content: + - type: tool_use + id: toolu_014m6v2DAw2eqNmpBfuTRTmq + name: HassTurnOn + input: + domain: cover + name: Garage Door Opener + area: Garage + - role: user + content: + - type: tool_result + tool_use_id: toolu_014m6v2DAw2eqNmpBfuTRTmq + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Garage", "type": "area", "id": "garage"}, {"name": + "Garage Door Opener", "type": "entity", "id": "cover.garage_door_opener"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The garage door has been opened. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + domain: cover + name: Garage Door Opener + area: Garage diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-lock_smart_lock.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-lock_smart_lock.yaml new file mode 100644 index 00000000..d9a08aa2 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-lock_smart_lock.yaml @@ -0,0 +1,74 @@ +--- +uuid: 840fb984-5276-41fa-9a78-35649e209362 +task_id: home1_us_lock_smart_lock-lock_smart_lock +category: lock +task: + input_text: Lock smart lock + expect_changes: + lock.smart_lock: + state: locked + attributes: null +response: The smart lock has been locked. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Lock smart lock + context: + id: 01J6SP7AEA0H0XN5XHF9BAB6PA + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:38:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Lock\n domain: binary_sensor\n state: 'off'\n\ + \ areas: Entry\n attributes:\n device_class: lock\n- names: Smart Lock\ + \ Tamper\n domain: binary_sensor\n state: 'off'\n areas: Entry\n attributes:\n\ + \ device_class: tamper\n- names: Smart Lock Battery\n domain: binary_sensor\n\ + \ state: 'off'\n areas: Entry\n attributes:\n device_class: battery\n\ + - names: Smart Lock\n domain: lock\n state: unlocked\n areas: Entry\n-\ + \ names: Smart Lock Battery\n domain: sensor\n state: '90'\n areas: Entry\n\ + \ attributes:\n unit_of_measurement: '%'\n device_class: battery\n" + messages: + - role: user + content: Lock smart lock + - role: assistant + content: + - type: tool_use + id: toolu_01A92PpQUa9wyUxYHUUw17Ld + name: HassTurnOn + input: + name: Smart Lock + domain: lock + area: Entry + - role: user + content: + - type: tool_result + tool_use_id: toolu_01A92PpQUa9wyUxYHUUw17Ld + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Entry", "type": "area", "id": "entry"}, {"name": + "Smart Lock", "type": "entity", "id": "lock.smart_lock"}], "failed": []}}' + - role: assistant + content: + - type: text + text: The smart lock has been locked. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Smart Lock + domain: lock + area: Entry diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-lock_the_entry_lock.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-lock_the_entry_lock.yaml new file mode 100644 index 00000000..a38b5926 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-lock_the_entry_lock.yaml @@ -0,0 +1,72 @@ +--- +uuid: a310eb30-e1a3-4c67-817b-410363941260 +task_id: home1_us_lock_smart_lock-lock_the_entry_lock +category: lock +task: + input_text: Lock the entry lock + expect_changes: + lock.smart_lock: + state: locked + attributes: null +response: The smart lock in the entry area has been locked. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Lock the entry lock + context: + id: 01J6SQBVRK8739ZNAE70S9R6XC + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:58:56. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Lock\n domain: binary_sensor\n state: 'off'\n\ + \ areas: Entry\n attributes:\n device_class: lock\n- names: Smart Lock\ + \ Tamper\n domain: binary_sensor\n state: 'off'\n areas: Entry\n attributes:\n\ + \ device_class: tamper\n- names: Smart Lock Battery\n domain: binary_sensor\n\ + \ state: 'off'\n areas: Entry\n attributes:\n device_class: battery\n\ + - names: Smart Lock Battery\n domain: sensor\n state: '90'\n areas: Entry\n\ + \ attributes:\n unit_of_measurement: '%'\n device_class: battery\n\ + - names: Smart Lock\n domain: lock\n state: unlocked\n areas: Entry\n" + messages: + - role: user + content: Lock the entry lock + - role: assistant + content: + - type: tool_use + id: toolu_01C4GnvZnEbvA9dkjkUezHNN + name: HassTurnOn + input: + name: Smart Lock + domain: lock + - role: user + content: + - type: tool_result + tool_use_id: toolu_01C4GnvZnEbvA9dkjkUezHNN + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Smart Lock", "type": "entity", "id": "lock.smart_lock"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The smart lock in the entry area has been locked. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Smart Lock + domain: lock diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-unlock_smart_lock.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-unlock_smart_lock.yaml new file mode 100644 index 00000000..aa65fd4c --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-unlock_smart_lock.yaml @@ -0,0 +1,72 @@ +--- +uuid: d123db88-f635-43ef-bc22-9425a5b9ae4a +task_id: home1_us_lock_smart_lock-unlock_smart_lock +category: lock +task: + input_text: Unlock smart lock + expect_changes: + lock.smart_lock: + state: unlocked + attributes: null +response: The smart lock has been unlocked. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Unlock smart lock + context: + id: 01J6SP988FFD7MYZ3SGXPJBBKG + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:40:02. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Lock\n domain: binary_sensor\n state: 'off'\n\ + \ areas: Entry\n attributes:\n device_class: lock\n- names: Smart Lock\ + \ Tamper\n domain: binary_sensor\n state: 'off'\n areas: Entry\n attributes:\n\ + \ device_class: tamper\n- names: Smart Lock Battery\n domain: binary_sensor\n\ + \ state: 'off'\n areas: Entry\n attributes:\n device_class: battery\n\ + - names: Smart Lock\n domain: lock\n state: locked\n areas: Entry\n- names:\ + \ Smart Lock Battery\n domain: sensor\n state: '90'\n areas: Entry\n attributes:\n\ + \ unit_of_measurement: '%'\n device_class: battery\n" + messages: + - role: user + content: Unlock smart lock + - role: assistant + content: + - type: tool_use + id: toolu_01MUXwWSp7vPCvCpveo96WEN + name: HassTurnOff + input: + name: Smart Lock + domain: lock + - role: user + content: + - type: tool_result + tool_use_id: toolu_01MUXwWSp7vPCvCpveo96WEN + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Smart Lock", "type": "entity", "id": "lock.smart_lock"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The smart lock has been unlocked. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + name: Smart Lock + domain: lock diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-unlock_the_entry_lock.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-unlock_the_entry_lock.yaml new file mode 100644 index 00000000..2a1b8ff4 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_lock_smart_lock-unlock_the_entry_lock.yaml @@ -0,0 +1,72 @@ +--- +uuid: b351af8b-e6d9-4107-900b-0f06557bc43c +task_id: home1_us_lock_smart_lock-unlock_the_entry_lock +category: lock +task: + input_text: Unlock the entry lock + expect_changes: + lock.smart_lock: + state: unlocked + attributes: null +response: The entry lock has been unlocked. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Unlock the entry lock + context: + id: 01J6SQC10D8739ZNAE70S9R6XC + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:59:01. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Lock\n domain: binary_sensor\n state: 'off'\n\ + \ areas: Entry\n attributes:\n device_class: lock\n- names: Smart Lock\ + \ Tamper\n domain: binary_sensor\n state: 'off'\n areas: Entry\n attributes:\n\ + \ device_class: tamper\n- names: Smart Lock Battery\n domain: binary_sensor\n\ + \ state: 'off'\n areas: Entry\n attributes:\n device_class: battery\n\ + - names: Smart Lock\n domain: lock\n state: locked\n areas: Entry\n- names:\ + \ Smart Lock Battery\n domain: sensor\n state: '90'\n areas: Entry\n attributes:\n\ + \ unit_of_measurement: '%'\n device_class: battery\n" + messages: + - role: user + content: Unlock the entry lock + - role: assistant + content: + - type: tool_use + id: toolu_01KEDZuJ3qcoPz7PxtxcfUno + name: HassTurnOff + input: + name: Smart Lock + domain: lock + - role: user + content: + - type: tool_result + tool_use_id: toolu_01KEDZuJ3qcoPz7PxtxcfUno + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Smart Lock", "type": "entity", "id": "lock.smart_lock"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The entry lock has been unlocked. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + name: Smart Lock + domain: lock diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-return_roborock_downstairs_to_base.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-return_roborock_downstairs_to_base.yaml new file mode 100644 index 00000000..80b0ee51 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-return_roborock_downstairs_to_base.yaml @@ -0,0 +1,73 @@ +--- +uuid: 83d36f55-534d-494a-a28c-9b5c3ab9f6af +task_id: home1_us_vacuum_vacuum-return_roborock_downstairs_to_base +category: vacuum +task: + input_text: Return Roborock Downstairs to base + expect_changes: + vacuum.roborock_downstairs: + state: returning + attributes: null +response: Okay, I've sent a command to return the Roborock Downstairs vacuum cleaner + in the Living Room back to its base. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Return Roborock Downstairs to base + context: + id: 01J6SP5FGHVH6BJNY8MW320DJN + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:37:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Roborock Downstairs\n domain: vacuum\n state: cleaning\n\ + \ areas: Living Room\n" + messages: + - role: user + content: Return Roborock Downstairs to base + - role: assistant + content: + - type: tool_use + id: toolu_016MDXzGNHcigAXAVBdZESDR + name: HassVacuumReturnToBase + input: + area: Living Room + domain: + - vacuum + name: Roborock Downstairs + - role: user + content: + - type: tool_result + tool_use_id: toolu_016MDXzGNHcigAXAVBdZESDR + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room", "type": "area", "id": "living_room"}, + {"name": "Roborock Downstairs", "type": "entity", "id": "vacuum.roborock_downstairs"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: Okay, I've sent a command to return the Roborock Downstairs vacuum + cleaner in the Living Room back to its base. + - event_type: tool_call + data: + tool_name: HassVacuumReturnToBase + tool_args: + area: Living Room + domain: + - vacuum + name: Roborock Downstairs diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-return_vacuum_in_the_living_room_to_base.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-return_vacuum_in_the_living_room_to_base.yaml new file mode 100644 index 00000000..23edf854 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-return_vacuum_in_the_living_room_to_base.yaml @@ -0,0 +1,69 @@ +--- +uuid: cb01cf90-ad46-47da-97dc-0eb90280090a +task_id: home1_us_vacuum_vacuum-return_vacuum_in_the_living_room_to_base +category: vacuum +task: + input_text: Return vacuum in the living room to base + expect_changes: + vacuum.roborock_downstairs: + state: returning + attributes: null +response: Returning the vacuum in the living room back to its base. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Return vacuum in the living room to base + context: + id: 01J6SQEC4FWYFDBM3YANJAH6T2 + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:00:18. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Roborock Downstairs\n domain: vacuum\n state: cleaning\n\ + \ areas: Living Room\n" + messages: + - role: user + content: Return vacuum in the living room to base + - role: assistant + content: + - type: tool_use + id: toolu_015g9uB1zBpsmhWUktFcUeXk + name: HassVacuumReturnToBase + input: + area: Living Room + domain: + - vacuum + - role: user + content: + - type: tool_result + tool_use_id: toolu_015g9uB1zBpsmhWUktFcUeXk + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room", "type": "area", "id": "living_room"}, + {"name": "Roborock Downstairs", "type": "entity", "id": "vacuum.roborock_downstairs"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: Returning the vacuum in the living room back to its base. + - event_type: tool_call + data: + tool_name: HassVacuumReturnToBase + tool_args: + area: Living Room + domain: + - vacuum diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-start_roborock_downstairs_vacuum.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-start_roborock_downstairs_vacuum.yaml new file mode 100644 index 00000000..c8002781 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-start_roborock_downstairs_vacuum.yaml @@ -0,0 +1,71 @@ +--- +uuid: cd49c96e-49cd-4f95-ab37-4af824669259 +task_id: home1_us_vacuum_vacuum-start_roborock_downstairs_vacuum +category: vacuum +task: + input_text: Start Roborock Downstairs vacuum + expect_changes: + vacuum.roborock_downstairs: + state: cleaning + attributes: null +response: The Roborock Downstairs vacuum in the Living Room area has been started. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Start Roborock Downstairs vacuum + context: + id: 01J6SR8RMJEVC7WJHENTWJ76BX + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:14:43. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Roborock Downstairs\n domain: vacuum\n state: 'off'\n\ + \ areas: Living Room\n" + messages: + - role: user + content: Start Roborock Downstairs vacuum + - role: assistant + content: + - type: tool_use + id: toolu_01HmyLr6B6PNg2Pv8YXm9sCq + name: HassVacuumStart + input: + name: Roborock Downstairs + domain: + - vacuum + area: Living Room + - role: user + content: + - type: tool_result + tool_use_id: toolu_01HmyLr6B6PNg2Pv8YXm9sCq + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room", "type": "area", "id": "living_room"}, + {"name": "Roborock Downstairs", "type": "entity", "id": "vacuum.roborock_downstairs"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The Roborock Downstairs vacuum in the Living Room area has been started. + - event_type: tool_call + data: + tool_name: HassVacuumStart + tool_args: + name: Roborock Downstairs + domain: + - vacuum + area: Living Room diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-start_vacuum_in_the_living_room.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-start_vacuum_in_the_living_room.yaml new file mode 100644 index 00000000..1e86bf47 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home1_us_vacuum_vacuum-start_vacuum_in_the_living_room.yaml @@ -0,0 +1,69 @@ +--- +uuid: 393e43ff-efef-4bf7-9b63-9f1cb823e28d +task_id: home1_us_vacuum_vacuum-start_vacuum_in_the_living_room +category: vacuum +task: + input_text: Start vacuum in the living room + expect_changes: + vacuum.roborock_downstairs: + state: cleaning + attributes: null +response: I've started the vacuum in the Living Room. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Start vacuum in the living room + context: + id: 01J6SP3N9TJXYHT804APG1WAKN + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:36:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Roborock Downstairs\n domain: vacuum\n state: 'off'\n\ + \ areas: Living Room\n" + messages: + - role: user + content: Start vacuum in the living room + - role: assistant + content: + - type: tool_use + id: toolu_01NPHWQJKSvf69NBQNAF7f7d + name: HassVacuumStart + input: + area: Living Room + domain: + - vacuum + - role: user + content: + - type: tool_result + tool_use_id: toolu_01NPHWQJKSvf69NBQNAF7f7d + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room", "type": "area", "id": "living_room"}, + {"name": "Roborock Downstairs", "type": "entity", "id": "vacuum.roborock_downstairs"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I've started the vacuum in the Living Room. + - event_type: tool_call + data: + tool_name: HassVacuumStart + tool_args: + area: Living Room + domain: + - vacuum diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-close_the_front_yard_valve.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-close_the_front_yard_valve.yaml new file mode 100644 index 00000000..ee76cd86 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-close_the_front_yard_valve.yaml @@ -0,0 +1,75 @@ +--- +uuid: 02552cef-a021-49c8-91bf-fcd91404ff14 +task_id: home2_ru_valve_water_valve-close_the_front_yard_valve +category: valve +task: + input_text: close the front yard valve + expect_changes: + valve.irrigation_valve: + state: closed + attributes: + current_position: 0 +response: The front yard valve has been closed. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: close the front yard valve + context: + id: 01J6SQFV19WGH6G0E7RN51VQ5Q + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:01:06. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Irrigation Valve Meter-reading\n domain: sensor\n\ + \ state: '2000'\n areas: Front yard\n attributes:\n unit_of_measurement:\ + \ m\xB3\n device_class: water\n- names: Irrigation Valve Battery\n domain:\ + \ sensor\n state: '75'\n areas: Front yard\n attributes:\n unit_of_measurement:\ + \ '%'\n device_class: battery\n- names: Irrigation Valve\n domain: valve\n\ + \ state: open\n areas: Front yard\n attributes:\n current_position:\ + \ '100'\n" + messages: + - role: user + content: close the front yard valve + - role: assistant + content: + - type: tool_use + id: toolu_01AM9NUZFPfNci58tRd7jGyU + name: HassTurnOff + input: + domain: + - valve + area: Front yard + - role: user + content: + - type: tool_result + tool_use_id: toolu_01AM9NUZFPfNci58tRd7jGyU + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Front yard", "type": "area", "id": "front_yard"}, + {"name": "Irrigation Valve", "type": "entity", "id": "valve.irrigation_valve"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The front yard valve has been closed. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + domain: + - valve + area: Front yard diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-close_the_irrigation_valve.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-close_the_irrigation_valve.yaml new file mode 100644 index 00000000..b0367434 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-close_the_irrigation_valve.yaml @@ -0,0 +1,76 @@ +--- +uuid: 69fcf854-13f8-49b8-9f64-fb00a767caa2 +task_id: home2_ru_valve_water_valve-close_the_irrigation_valve +category: valve +task: + input_text: close the irrigation valve + expect_changes: + valve.irrigation_valve: + state: closed + attributes: + current_position: 0 +response: The irrigation valve has been closed. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: close the irrigation valve + context: + id: 01J6SQ0Y32VF2C7BBZ0TPS0QGY + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:52:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Irrigation Valve\n domain: valve\n state: open\n\ + \ areas: Front yard\n attributes:\n current_position: '100'\n- names:\ + \ Irrigation Valve Meter-reading\n domain: sensor\n state: '2000'\n areas:\ + \ Front yard\n attributes:\n unit_of_measurement: m\xB3\n device_class:\ + \ water\n- names: Irrigation Valve Battery\n domain: sensor\n state: '75'\n\ + \ areas: Front yard\n attributes:\n unit_of_measurement: '%'\n device_class:\ + \ battery\n" + messages: + - role: user + content: close the irrigation valve + - role: assistant + content: + - type: text + text: 'Okay, here is the command to close the irrigation valve:' + - type: tool_use + id: toolu_01DuiBD6CKBNtLYoji5Hckcb + name: HassTurnOff + input: + domain: + - valve + name: Irrigation Valve + - role: user + content: + - type: tool_result + tool_use_id: toolu_01DuiBD6CKBNtLYoji5Hckcb + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Irrigation Valve", "type": "entity", "id": "valve.irrigation_valve"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The irrigation valve has been closed. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + domain: + - valve + name: Irrigation Valve diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-open_the_front_yard_valve.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-open_the_front_yard_valve.yaml new file mode 100644 index 00000000..b6368c90 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-open_the_front_yard_valve.yaml @@ -0,0 +1,75 @@ +--- +uuid: 24a9f0a5-fb7c-4e05-b8c2-95d026fe4948 +task_id: home2_ru_valve_water_valve-open_the_front_yard_valve +category: valve +task: + input_text: open the front yard valve + expect_changes: + valve.irrigation_valve: + state: open + attributes: + current_position: 100 +response: I have turned on the Irrigation Valve in the Front yard area. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: open the front yard valve + context: + id: 01J6SR27KCWGH6G0E7RN51VQ5Q + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:11:09. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Irrigation Valve Meter-reading\n domain: sensor\n\ + \ state: '2000'\n areas: Front yard\n attributes:\n unit_of_measurement:\ + \ m\xB3\n device_class: water\n- names: Irrigation Valve Battery\n domain:\ + \ sensor\n state: '75'\n areas: Front yard\n attributes:\n unit_of_measurement:\ + \ '%'\n device_class: battery\n- names: Irrigation Valve\n domain: valve\n\ + \ state: closed\n areas: Front yard\n attributes:\n current_position:\ + \ '0'\n" + messages: + - role: user + content: open the front yard valve + - role: assistant + content: + - type: tool_use + id: toolu_01SRwpizdHf7LcbF8vf6n2YY + name: HassTurnOn + input: + name: Irrigation Valve + domain: valve + area: Front yard + - role: user + content: + - type: tool_result + tool_use_id: toolu_01SRwpizdHf7LcbF8vf6n2YY + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Front yard", "type": "area", "id": "front_yard"}, + {"name": "Irrigation Valve", "type": "entity", "id": "valve.irrigation_valve"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I have turned on the Irrigation Valve in the Front yard area. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Irrigation Valve + domain: valve + area: Front yard diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-open_the_irrigation_valve.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-open_the_irrigation_valve.yaml new file mode 100644 index 00000000..0d4fee38 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-open_the_irrigation_valve.yaml @@ -0,0 +1,72 @@ +--- +uuid: 94c38992-19be-46cb-bee3-ed7e21cafeed +task_id: home2_ru_valve_water_valve-open_the_irrigation_valve +category: valve +task: + input_text: open the irrigation valve + expect_changes: + valve.irrigation_valve: + state: open + attributes: + current_position: 100 +response: The irrigation valve has been turned on. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: open the irrigation valve + context: + id: 01J6SPZ3ECS05KKKMVEAF3804W + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:51:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Irrigation Valve Meter-reading\n domain: sensor\n\ + \ state: '2000'\n areas: Front yard\n attributes:\n unit_of_measurement:\ + \ m\xB3\n device_class: water\n- names: Irrigation Valve Battery\n domain:\ + \ sensor\n state: '75'\n areas: Front yard\n attributes:\n unit_of_measurement:\ + \ '%'\n device_class: battery\n- names: Irrigation Valve\n domain: valve\n\ + \ state: closed\n areas: Front yard\n attributes:\n current_position:\ + \ '0'\n" + messages: + - role: user + content: open the irrigation valve + - role: assistant + content: + - type: tool_use + id: toolu_01JTRRmGPfD39JHidajwRHJE + name: HassTurnOn + input: + domain: valve + name: Irrigation Valve + - role: user + content: + - type: tool_result + tool_use_id: toolu_01JTRRmGPfD39JHidajwRHJE + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Irrigation Valve", "type": "entity", "id": "valve.irrigation_valve"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The irrigation valve has been turned on. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + domain: valve + name: Irrigation Valve diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-set_the_irrigation_valve_to_50.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-set_the_irrigation_valve_to_50.yaml new file mode 100644 index 00000000..d22c51e9 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home2_ru_valve_water_valve-set_the_irrigation_valve_to_50.yaml @@ -0,0 +1,74 @@ +--- +uuid: 631ec180-3008-49b0-9a13-a13e7fc8cf87 +task_id: home2_ru_valve_water_valve-set_the_irrigation_valve_to_50 +category: valve +task: + input_text: set the irrigation valve to 50% + expect_changes: + valve.irrigation_valve: + state: open + attributes: + current_position: 50 +response: I've set the Irrigation Valve to 50% position. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: set the irrigation valve to 50% + context: + id: 01J6SQHAS0WGH6G0E7RN51VQ5Q + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:01:55. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Irrigation Valve Meter-reading\n domain: sensor\n\ + \ state: '2000'\n areas: Front yard\n attributes:\n unit_of_measurement:\ + \ m\xB3\n device_class: water\n- names: Irrigation Valve Battery\n domain:\ + \ sensor\n state: '75'\n areas: Front yard\n attributes:\n unit_of_measurement:\ + \ '%'\n device_class: battery\n- names: Irrigation Valve\n domain: valve\n\ + \ state: open\n areas: Front yard\n attributes:\n current_position:\ + \ '100'\n" + messages: + - role: user + content: set the irrigation valve to 50% + - role: assistant + content: + - type: tool_use + id: toolu_01E8sp88kMd27K1u1yUgYgXC + name: HassSetPosition + input: + name: Irrigation Valve + domain: valve + position: 50 + - role: user + content: + - type: tool_result + tool_use_id: toolu_01E8sp88kMd27K1u1yUgYgXC + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Irrigation Valve", "type": "entity", "id": "valve.irrigation_valve"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I've set the Irrigation Valve to 50% position. + - event_type: tool_call + data: + tool_name: HassSetPosition + tool_args: + name: Irrigation Valve + domain: valve + position: 50 diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_off_the_bedroom_1_fan.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_off_the_bedroom_1_fan.yaml new file mode 100644 index 00000000..26f06dca --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_off_the_bedroom_1_fan.yaml @@ -0,0 +1,70 @@ +--- +uuid: bdcacfed-f486-42a9-9998-b53a0c946685 +task_id: home5_cn_fan_fan-turn_off_the_bedroom_1_fan +category: fan +task: + input_text: Turn off the bedroom 1 fan + expect_changes: + fan.bedroom_fan: + state: 'off' + attributes: + percentage: 0 +response: The Bedroom 1 fan has been turned off. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Turn off the bedroom 1 fan + context: + id: 01J6SQHFN6EVC7WJHENTWJ76BX + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:02:00. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Bedroom Fan\n domain: fan\n state: 'on'\n areas:\ + \ Bedroom 1\n attributes:\n percentage: '100'\n" + messages: + - role: user + content: Turn off the bedroom 1 fan + - role: assistant + content: + - type: tool_use + id: toolu_01SSHVwu1g3fvDY74DQZ5uAF + name: HassTurnOff + input: + area: Bedroom 1 + domain: + - fan + - role: user + content: + - type: tool_result + tool_use_id: toolu_01SSHVwu1g3fvDY74DQZ5uAF + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Bedroom 1", "type": "area", "id": "bedroom_1"}, + {"name": "Bedroom Fan", "type": "entity", "id": "fan.bedroom_fan"}], "failed": + []}}' + - role: assistant + content: + - type: text + text: The Bedroom 1 fan has been turned off. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + area: Bedroom 1 + domain: + - fan diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_off_the_bedroom_fan.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_off_the_bedroom_fan.yaml new file mode 100644 index 00000000..3de07513 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_off_the_bedroom_fan.yaml @@ -0,0 +1,67 @@ +--- +uuid: 5f970724-03c5-442e-bd4d-861d792ddd96 +task_id: home5_cn_fan_fan-turn_off_the_bedroom_fan +category: fan +task: + input_text: Turn off the bedroom fan + expect_changes: + fan.bedroom_fan: + state: 'off' + attributes: + percentage: 0 +response: I've turned off the Bedroom Fan. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Turn off the bedroom fan + context: + id: 01J6SP1TEPR0H4MGES7QK1CDC5 + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:35:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Bedroom Fan\n domain: fan\n state: 'on'\n areas:\ + \ Bedroom 1\n attributes:\n percentage: '100'\n" + messages: + - role: user + content: Turn off the bedroom fan + - role: assistant + content: + - type: tool_use + id: toolu_01YaLh5PkC9m6WM9AQM4q4S8 + name: HassTurnOff + input: + name: Bedroom Fan + domain: fan + - role: user + content: + - type: tool_result + tool_use_id: toolu_01YaLh5PkC9m6WM9AQM4q4S8 + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Bedroom Fan", "type": "entity", "id": "fan.bedroom_fan"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I've turned off the Bedroom Fan. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + name: Bedroom Fan + domain: fan diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_on_the_bedroom_1_fan.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_on_the_bedroom_1_fan.yaml new file mode 100644 index 00000000..df7ced80 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_on_the_bedroom_1_fan.yaml @@ -0,0 +1,72 @@ +--- +uuid: 9f7406b5-9a96-4d62-98de-d0135b0407b8 +task_id: home5_cn_fan_fan-turn_on_the_bedroom_1_fan +category: fan +task: + input_text: Turn on the bedroom 1 fan + expect_changes: + fan.bedroom_fan: + state: 'on' + attributes: + percentage: 100 +response: The bedroom 1 fan has been turned on. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Turn on the bedroom 1 fan + context: + id: 01J6SNZZVGP0D28AH331N34G0N + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:34:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Bedroom Fan\n domain: fan\n state: 'off'\n areas:\ + \ Bedroom 1\n attributes:\n percentage: '0'\n" + messages: + - role: user + content: Turn on the bedroom 1 fan + - role: assistant + content: + - type: tool_use + id: toolu_01TvqgTFqyVE6YX5XG9UWpxx + name: HassTurnOn + input: + name: Bedroom Fan + domain: + - fan + area: Bedroom 1 + - role: user + content: + - type: tool_result + tool_use_id: toolu_01TvqgTFqyVE6YX5XG9UWpxx + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Bedroom 1", "type": "area", "id": "bedroom_1"}, + {"name": "Bedroom Fan", "type": "entity", "id": "fan.bedroom_fan"}], "failed": + []}}' + - role: assistant + content: + - type: text + text: The bedroom 1 fan has been turned on. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Bedroom Fan + domain: + - fan + area: Bedroom 1 diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_on_the_bedroom_fan.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_on_the_bedroom_fan.yaml new file mode 100644 index 00000000..af4d5f9e --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_fan_fan-turn_on_the_bedroom_fan.yaml @@ -0,0 +1,67 @@ +--- +uuid: e62b5aa8-2d5a-42f8-90e5-551b4c1ad5fd +task_id: home5_cn_fan_fan-turn_on_the_bedroom_fan +category: fan +task: + input_text: Turn on the bedroom fan + expect_changes: + fan.bedroom_fan: + state: 'on' + attributes: + percentage: 100 +response: The bedroom fan has been turned on. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Turn on the bedroom fan + context: + id: 01J6SQKV18TPW2TE3Z3RHNPMQD + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:03:17. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Bedroom Fan\n domain: fan\n state: 'off'\n areas:\ + \ Bedroom 1\n attributes:\n percentage: '0'\n" + messages: + - role: user + content: Turn on the bedroom fan + - role: assistant + content: + - type: tool_use + id: toolu_01A35Ei7uVSJs1E4dpZGKujL + name: HassTurnOn + input: + name: Bedroom Fan + domain: fan + - role: user + content: + - type: tool_result + tool_use_id: toolu_01A35Ei7uVSJs1E4dpZGKujL + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Bedroom Fan", "type": "entity", "id": "fan.bedroom_fan"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The bedroom fan has been turned on. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Bedroom Fan + domain: fan diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_light_light-set_the_living_room_light_to_50_brightness.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_light_light-set_the_living_room_light_to_50_brightness.yaml new file mode 100644 index 00000000..72b76144 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home5_cn_light_light-set_the_living_room_light_to_50_brightness.yaml @@ -0,0 +1,74 @@ +--- +uuid: 292efda4-6b74-4816-9b45-21e6754a56a8 +task_id: home5_cn_light_light-set_the_living_room_light_to_50_brightness +category: light +task: + input_text: Set the living room light to 50% brightness + expect_changes: + light.living_room_light: + state: 'on' + attributes: + brightness: 128 + color_mode: brightness +response: The living room light has been set to 50% brightness. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Set the living room light to 50% brightness + context: + id: 01J6SREVTZ3HY8436FCAH76939 + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:18:03. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Living Room Light\n domain: light\n state: 'off'\n\ + \ areas: Living Room\n attributes:\n brightness:\n- names: Kitchenette\ + \ Light\n domain: light\n state: 'off'\n areas: Kitchenette\n- names: Bedroom\ + \ 1 Light\n domain: light\n state: 'off'\n areas: Bedroom 1\n- names: Bedroom\ + \ 2 Light\n domain: light\n state: 'off'\n areas: Bedroom 2\n- names: Porch\ + \ Light\n domain: light\n state: 'off'\n areas: Front Porch\n" + messages: + - role: user + content: Set the living room light to 50% brightness + - role: assistant + content: + - type: tool_use + id: toolu_01G7fJK6UGPv7qP7QyvjaedJ + name: HassLightSet + input: + name: Living Room Light + domain: light + brightness: 50 + - role: user + content: + - type: tool_result + tool_use_id: toolu_01G7fJK6UGPv7qP7QyvjaedJ + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room Light", "type": "entity", "id": + "light.living_room_light"}], "failed": []}}' + - role: assistant + content: + - type: text + text: The living room light has been set to 50% brightness. + - event_type: tool_call + data: + tool_name: HassLightSet + tool_args: + name: Living Room Light + domain: light + brightness: 50 diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-close_the_living_room_curtains.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-close_the_living_room_curtains.yaml new file mode 100644 index 00000000..8b7aa835 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-close_the_living_room_curtains.yaml @@ -0,0 +1,72 @@ +--- +uuid: 5aec9fd7-02d6-4189-9316-a8e636274897 +task_id: home7_dk_cover_curtain_cover_curtain-close_the_living_room_curtains +category: cover +task: + input_text: Close the living room curtains + expect_changes: + cover.smart_curtain: + state: closed + attributes: null +response: The living room curtains have been closed. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Close the living room curtains + context: + id: 01J6SRC01S4R91Q9AW95A1EV7E + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:16:29. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Curtain\n domain: cover\n state: open\n areas:\ + \ Living Room\n attributes:\n current_position: '100'\n device_class:\ + \ curtain\n- names: Smart Curtain Battery\n domain: sensor\n state: '90'\n\ + \ areas: Living Room\n attributes:\n unit_of_measurement: '%'\n device_class:\ + \ battery\n" + messages: + - role: user + content: Close the living room curtains + - role: assistant + content: + - type: tool_use + id: toolu_01EQgxS7UzdoqszwbTpEnEPQ + name: HassTurnOff + input: + name: Smart Curtain + domain: cover + area: Living Room + - role: user + content: + - type: tool_result + tool_use_id: toolu_01EQgxS7UzdoqszwbTpEnEPQ + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room", "type": "area", "id": "living_room"}, + {"name": "Smart Curtain", "type": "entity", "id": "cover.smart_curtain"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The living room curtains have been closed. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + name: Smart Curtain + domain: cover + area: Living Room diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-close_the_living_room_smart_curtain.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-close_the_living_room_smart_curtain.yaml new file mode 100644 index 00000000..b710ba43 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-close_the_living_room_smart_curtain.yaml @@ -0,0 +1,72 @@ +--- +uuid: 07ec7170-bf35-4780-96ce-d30894c86e78 +task_id: home7_dk_cover_curtain_cover_curtain-close_the_living_room_smart_curtain +category: cover +task: + input_text: Close the living room smart curtain + expect_changes: + cover.smart_curtain: + state: closed + attributes: null +response: Closing the living room smart curtain. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Close the living room smart curtain + context: + id: 01J6SPJ9W5D7SRFGRTCW8G3N6K + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:44:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Curtain Battery\n domain: sensor\n state:\ + \ '90'\n areas: Living Room\n attributes:\n unit_of_measurement: '%'\n\ + \ device_class: battery\n- names: Smart Curtain\n domain: cover\n state:\ + \ open\n areas: Living Room\n attributes:\n current_position: '100'\n\ + \ device_class: curtain\n" + messages: + - role: user + content: Close the living room smart curtain + - role: assistant + content: + - type: tool_use + id: toolu_01YDBVWoSWnNqYPwQ6p4QmoQ + name: HassTurnOff + input: + name: Smart Curtain + domain: cover + area: Living Room + - role: user + content: + - type: tool_result + tool_use_id: toolu_01YDBVWoSWnNqYPwQ6p4QmoQ + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room", "type": "area", "id": "living_room"}, + {"name": "Smart Curtain", "type": "entity", "id": "cover.smart_curtain"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: Closing the living room smart curtain. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + name: Smart Curtain + domain: cover + area: Living Room diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-close_the_smart_cutains.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-close_the_smart_cutains.yaml new file mode 100644 index 00000000..a67df265 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-close_the_smart_cutains.yaml @@ -0,0 +1,74 @@ +--- +uuid: 9903874c-e990-470c-943d-9f32bcf2950d +task_id: home7_dk_cover_curtain_cover_curtain-close_the_smart_cutains +category: cover +task: + input_text: Close the smart cutains + expect_changes: + cover.smart_curtain: + state: closed + attributes: null +response: The smart curtains in the Living Room have been closed. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Close the smart cutains + context: + id: 01J6SQNC7Q4R91Q9AW95A1EV7E + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:04:08. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Curtain\n domain: cover\n state: open\n areas:\ + \ Living Room\n attributes:\n current_position: '100'\n device_class:\ + \ curtain\n- names: Smart Curtain Battery\n domain: sensor\n state: '90'\n\ + \ areas: Living Room\n attributes:\n unit_of_measurement: '%'\n device_class:\ + \ battery\n" + messages: + - role: user + content: Close the smart cutains + - role: assistant + content: + - type: text + text: Alright, let's close the smart curtains in the Living Room. + - type: tool_use + id: toolu_01LifwzSAdZ2RoyTDYzQn435 + name: HassTurnOff + input: + domain: + - cover + area: Living Room + - role: user + content: + - type: tool_result + tool_use_id: toolu_01LifwzSAdZ2RoyTDYzQn435 + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room", "type": "area", "id": "living_room"}, + {"name": "Smart Curtain", "type": "entity", "id": "cover.smart_curtain"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The smart curtains in the Living Room have been closed. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + domain: + - cover + area: Living Room diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-open_the_living_room_curtains.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-open_the_living_room_curtains.yaml new file mode 100644 index 00000000..a3d4ea49 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-open_the_living_room_curtains.yaml @@ -0,0 +1,72 @@ +--- +uuid: 3368571f-0364-4640-8949-be55c0c2a522 +task_id: home7_dk_cover_curtain_cover_curtain-open_the_living_room_curtains +category: cover +task: + input_text: Open the living room curtains + expect_changes: + cover.smart_curtain: + state: open + attributes: null +response: I've opened the smart curtains in the living room. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Open the living room curtains + context: + id: 01J6SPGGD70J63KGM5YDPF36RJ + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:43:59. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Curtain\n domain: cover\n state: closed\n\ + \ areas: Living Room\n attributes:\n current_position: '0'\n device_class:\ + \ curtain\n- names: Smart Curtain Battery\n domain: sensor\n state: '90'\n\ + \ areas: Living Room\n attributes:\n unit_of_measurement: '%'\n device_class:\ + \ battery\n" + messages: + - role: user + content: Open the living room curtains + - role: assistant + content: + - type: tool_use + id: toolu_012kHBLPD6yAWwWBeAXC3gAD + name: HassTurnOn + input: + name: Smart Curtain + domain: cover + area: Living Room + - role: user + content: + - type: tool_result + tool_use_id: toolu_012kHBLPD6yAWwWBeAXC3gAD + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room", "type": "area", "id": "living_room"}, + {"name": "Smart Curtain", "type": "entity", "id": "cover.smart_curtain"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I've opened the smart curtains in the living room. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Smart Curtain + domain: cover + area: Living Room diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-open_the_living_room_smart_curtain.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-open_the_living_room_smart_curtain.yaml new file mode 100644 index 00000000..f138e82d --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-open_the_living_room_smart_curtain.yaml @@ -0,0 +1,72 @@ +--- +uuid: e449e983-fd38-43c7-9aad-cb0c4dbf7938 +task_id: home7_dk_cover_curtain_cover_curtain-open_the_living_room_smart_curtain +category: cover +task: + input_text: Open the living room smart curtain + expect_changes: + cover.smart_curtain: + state: open + attributes: null +response: The living room smart curtain has been opened. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Open the living room smart curtain + context: + id: 01J6SR3RZ44R91Q9AW95A1EV7E + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:11:59. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Curtain\n domain: cover\n state: closed\n\ + \ areas: Living Room\n attributes:\n current_position: '0'\n device_class:\ + \ curtain\n- names: Smart Curtain Battery\n domain: sensor\n state: '90'\n\ + \ areas: Living Room\n attributes:\n unit_of_measurement: '%'\n device_class:\ + \ battery\n" + messages: + - role: user + content: Open the living room smart curtain + - role: assistant + content: + - type: tool_use + id: toolu_01MUPTnwwnXAZzxMwQQBfuRq + name: HassTurnOn + input: + name: Smart Curtain + domain: cover + area: Living Room + - role: user + content: + - type: tool_result + tool_use_id: toolu_01MUPTnwwnXAZzxMwQQBfuRq + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room", "type": "area", "id": "living_room"}, + {"name": "Smart Curtain", "type": "entity", "id": "cover.smart_curtain"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The living room smart curtain has been opened. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Smart Curtain + domain: cover + area: Living Room diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-open_the_smart_curtain.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-open_the_smart_curtain.yaml new file mode 100644 index 00000000..f51a31a7 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-open_the_smart_curtain.yaml @@ -0,0 +1,69 @@ +--- +uuid: 2e8003b1-c655-438d-a436-fd1c4602d393 +task_id: home7_dk_cover_curtain_cover_curtain-open_the_smart_curtain +category: cover +task: + input_text: Open the smart curtain + expect_changes: + cover.smart_curtain: + state: open + attributes: null +response: I have opened the smart curtain. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Open the smart curtain + context: + id: 01J6SPE7S5GJXPSECSNMWPQKTT + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:42:45. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Curtain\n domain: cover\n state: closed\n\ + \ areas: Living Room\n attributes:\n current_position: '0'\n device_class:\ + \ curtain\n- names: Smart Curtain Battery\n domain: sensor\n state: '90'\n\ + \ areas: Living Room\n attributes:\n unit_of_measurement: '%'\n device_class:\ + \ battery\n" + messages: + - role: user + content: Open the smart curtain + - role: assistant + content: + - type: tool_use + id: toolu_01LYkJWUW2KjVNqLAxjZ9hRw + name: HassTurnOn + input: + name: Smart Curtain + domain: cover + - role: user + content: + - type: tool_result + tool_use_id: toolu_01LYkJWUW2KjVNqLAxjZ9hRw + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Smart Curtain", "type": "entity", "id": "cover.smart_curtain"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I have opened the smart curtain. + - event_type: tool_call + data: + tool_name: HassTurnOn + tool_args: + name: Smart Curtain + domain: cover diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-set_the_living_room_curtains_to_50.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-set_the_living_room_curtains_to_50.yaml new file mode 100644 index 00000000..4281460b --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-set_the_living_room_curtains_to_50.yaml @@ -0,0 +1,77 @@ +--- +uuid: 6b9de89f-db72-4707-96e7-6899538620ab +task_id: home7_dk_cover_curtain_cover_curtain-set_the_living_room_curtains_to_50 +category: cover +task: + input_text: Set the living room curtains to 50% + expect_changes: + cover.smart_curtain: + state: open + attributes: + current_position: 50 +response: The living room curtains have been set to 50% position. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Set the living room curtains to 50% + context: + id: 01J6SQPVCF4R91Q9AW95A1EV7E + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:04:56. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Curtain\n domain: cover\n state: closed\n\ + \ areas: Living Room\n attributes:\n current_position: '0'\n device_class:\ + \ curtain\n- names: Smart Curtain Battery\n domain: sensor\n state: '90'\n\ + \ areas: Living Room\n attributes:\n unit_of_measurement: '%'\n device_class:\ + \ battery\n" + messages: + - role: user + content: Set the living room curtains to 50% + - role: assistant + content: + - type: tool_use + id: toolu_01Snr7FkSGZPEqT6GE5qadXv + name: HassSetPosition + input: + name: Smart Curtain + domain: + - cover + area: Living Room + position: 50 + - role: user + content: + - type: tool_result + tool_use_id: toolu_01Snr7FkSGZPEqT6GE5qadXv + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Living Room", "type": "area", "id": "living_room"}, + {"name": "Smart Curtain", "type": "entity", "id": "cover.smart_curtain"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The living room curtains have been set to 50% position. + - event_type: tool_call + data: + tool_name: HassSetPosition + tool_args: + name: Smart Curtain + domain: + - cover + area: Living Room + position: 50 diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-set_the_smart_curtain_to_50.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-set_the_smart_curtain_to_50.yaml new file mode 100644 index 00000000..ea763289 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_cover_curtain_cover_curtain-set_the_smart_curtain_to_50.yaml @@ -0,0 +1,72 @@ +--- +uuid: 71af38b8-c816-4c80-aa45-f29b6790bfc8 +task_id: home7_dk_cover_curtain_cover_curtain-set_the_smart_curtain_to_50 +category: cover +task: + input_text: Set the smart curtain to 50% + expect_changes: + cover.smart_curtain: + state: open + attributes: + current_position: 50 +response: The smart curtain is now set to 50% position. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Set the smart curtain to 50% + context: + id: 01J6SPM59DYWXPGXHWP14R91ZY + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:45:59. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Smart Curtain\n domain: cover\n state: closed\n\ + \ areas: Living Room\n attributes:\n current_position: '0'\n device_class:\ + \ curtain\n- names: Smart Curtain Battery\n domain: sensor\n state: '90'\n\ + \ areas: Living Room\n attributes:\n unit_of_measurement: '%'\n device_class:\ + \ battery\n" + messages: + - role: user + content: Set the smart curtain to 50% + - role: assistant + content: + - type: tool_use + id: toolu_013x6azxj7ahZVDAHw9pJoVp + name: HassSetPosition + input: + name: Smart Curtain + domain: cover + position: 50 + - role: user + content: + - type: tool_result + tool_use_id: toolu_013x6azxj7ahZVDAHw9pJoVp + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Smart Curtain", "type": "entity", "id": "cover.smart_curtain"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The smart curtain is now set to 50% position. + - event_type: tool_call + data: + tool_name: HassSetPosition + tool_args: + name: Smart Curtain + domain: cover + position: 50 diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-mute_the_outdoor_speakers.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-mute_the_outdoor_speakers.yaml new file mode 100644 index 00000000..a41bccbe --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-mute_the_outdoor_speakers.yaml @@ -0,0 +1,75 @@ +--- +uuid: b456aaa6-f09a-450a-9953-8bc5a6e61993 +task_id: home7_dk_media_player_media_player-mute_the_outdoor_speakers +category: media-player +task: + input_text: Mute the outdoor speakers + expect_changes: + media_player.outdoor_speakers: + state: null + attributes: + volume_level: 0.0 +response: I've muted the Outdoor Speakers in the Rooftop Terrace area. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Mute the outdoor speakers + context: + id: 01J6SQQ0Z7EVC7WJHENTWJ76BX + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:05:02. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Outdoor Speakers\n domain: media_player\n state:\ + \ playing\n areas: Rooftop Terrace\n attributes:\n volume_level: 0.6\n\ + \ device_class: speaker\n" + messages: + - role: user + content: Mute the outdoor speakers + - role: assistant + content: + - type: tool_use + id: toolu_01U3S8rgf8wtMvAT5c4sytJ8 + name: HassSetVolume + input: + area: Rooftop Terrace + domain: + - media_player + name: Outdoor Speakers + volume_level: 0 + - role: user + content: + - type: tool_result + tool_use_id: toolu_01U3S8rgf8wtMvAT5c4sytJ8 + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Rooftop Terrace", "type": "area", "id": "rooftop_terrace"}, + {"name": "Outdoor Speakers", "type": "entity", "id": "media_player.outdoor_speakers"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I've muted the Outdoor Speakers in the Rooftop Terrace area. + - event_type: tool_call + data: + tool_name: HassSetVolume + tool_args: + area: Rooftop Terrace + domain: + - media_player + name: Outdoor Speakers + volume_level: 0 diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-pause_outdoor_speakers.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-pause_outdoor_speakers.yaml new file mode 100644 index 00000000..c2503d95 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-pause_outdoor_speakers.yaml @@ -0,0 +1,76 @@ +--- +uuid: 7d94d49c-3af6-4f08-80d0-3e1b96dc20f7 +task_id: home7_dk_media_player_media_player-pause_outdoor_speakers +category: media-player +task: + input_text: Pause Outdoor Speakers + expect_changes: + media_player.outdoor_speakers: + state: paused + attributes: null +response: The Outdoor Speakers in the Rooftop Terrace area have been paused. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Pause Outdoor Speakers + context: + id: 01J6SNNH6SEVC7WJHENTWJ76BX + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:29:16. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Outdoor Speakers\n domain: media_player\n state:\ + \ playing\n areas: Rooftop Terrace\n attributes:\n volume_level: 0.5\n\ + \ device_class: speaker\n" + messages: + - role: user + content: Pause Outdoor Speakers + - role: assistant + content: + - type: tool_use + id: toolu_019ogWHf7Ljr6RPKHkBKsooN + name: HassMediaPause + input: + area: Rooftop Terrace + device_class: + - speaker + domain: + - media_player + name: Outdoor Speakers + - role: user + content: + - type: tool_result + tool_use_id: toolu_019ogWHf7Ljr6RPKHkBKsooN + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Rooftop Terrace", "type": "area", "id": "rooftop_terrace"}, + {"name": "Outdoor Speakers", "type": "entity", "id": "media_player.outdoor_speakers"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The Outdoor Speakers in the Rooftop Terrace area have been paused. + - event_type: tool_call + data: + tool_name: HassMediaPause + tool_args: + area: Rooftop Terrace + device_class: + - speaker + domain: + - media_player + name: Outdoor Speakers diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-pause_the_rooftop_terrace_music.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-pause_the_rooftop_terrace_music.yaml new file mode 100644 index 00000000..8d0acdfe --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-pause_the_rooftop_terrace_music.yaml @@ -0,0 +1,70 @@ +--- +uuid: 19c7cdf1-cfd6-451e-ac27-1c20acfe59bb +task_id: home7_dk_media_player_media_player-pause_the_rooftop_terrace_music +category: media-player +task: + input_text: Pause the Rooftop Terrace music + expect_changes: + media_player.outdoor_speakers: + state: paused + attributes: null +response: The music playing on the Rooftop Terrace speakers has been paused. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Pause the Rooftop Terrace music + context: + id: 01J6SNPY5VM9Q05ZAWXCCM7K2A + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:30:02. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Outdoor Speakers\n domain: media_player\n state:\ + \ playing\n areas: Rooftop Terrace\n attributes:\n volume_level: 0.5\n\ + \ device_class: speaker\n" + messages: + - role: user + content: Pause the Rooftop Terrace music + - role: assistant + content: + - type: tool_use + id: toolu_01TSSVxSq2bzcQ2yn4AAxCgR + name: HassMediaPause + input: + area: Rooftop Terrace + domain: + - media_player + - role: user + content: + - type: tool_result + tool_use_id: toolu_01TSSVxSq2bzcQ2yn4AAxCgR + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Rooftop Terrace", "type": "area", "id": "rooftop_terrace"}, + {"name": "Outdoor Speakers", "type": "entity", "id": "media_player.outdoor_speakers"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The music playing on the Rooftop Terrace speakers has been paused. + - event_type: tool_call + data: + tool_name: HassMediaPause + tool_args: + area: Rooftop Terrace + domain: + - media_player diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-resume_outdoor_speakers.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-resume_outdoor_speakers.yaml new file mode 100644 index 00000000..f8d8dd8b --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-resume_outdoor_speakers.yaml @@ -0,0 +1,74 @@ +--- +uuid: 09543a11-aef6-41ed-b32c-bd5af0963a1e +task_id: home7_dk_media_player_media_player-resume_outdoor_speakers +category: media-player +task: + input_text: Resume Outdoor Speakers + expect_changes: + media_player.outdoor_speakers: + state: playing + attributes: null +response: The Outdoor Speakers on the Rooftop Terrace have been resumed. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Resume Outdoor Speakers + context: + id: 01J6SQSBK4HSPD6S1YAEWT5YTP + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:06:18. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Outdoor Speakers\n domain: media_player\n state:\ + \ paused\n areas: Rooftop Terrace\n attributes:\n volume_level: 0.5\n\ + \ device_class: speaker\n" + messages: + - role: user + content: Resume Outdoor Speakers + - role: assistant + content: + - type: text + text: 'Okay, here is the command to resume the Outdoor Speakers:' + - type: tool_use + id: toolu_01RN8bjPXyP2ES8oefCgjpY9 + name: HassMediaUnpause + input: + name: Outdoor Speakers + domain: + - media_player + area: Rooftop Terrace + - role: user + content: + - type: tool_result + tool_use_id: toolu_01RN8bjPXyP2ES8oefCgjpY9 + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Rooftop Terrace", "type": "area", "id": "rooftop_terrace"}, + {"name": "Outdoor Speakers", "type": "entity", "id": "media_player.outdoor_speakers"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The Outdoor Speakers on the Rooftop Terrace have been resumed. + - event_type: tool_call + data: + tool_name: HassMediaUnpause + tool_args: + name: Outdoor Speakers + domain: + - media_player + area: Rooftop Terrace diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-set_outdoor_speakers_volume_to_50.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-set_outdoor_speakers_volume_to_50.yaml new file mode 100644 index 00000000..5ad3c275 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-set_outdoor_speakers_volume_to_50.yaml @@ -0,0 +1,77 @@ +--- +uuid: 1d799e99-9db2-4fe8-94f4-80d0929ad088 +task_id: home7_dk_media_player_media_player-set_outdoor_speakers_volume_to_50 +category: media-player +task: + input_text: Set outdoor speakers volume to 50% + expect_changes: + media_player.outdoor_speakers: + state: null + attributes: + volume_level: 0.5 +response: I have adjusted the volume of the Outdoor Speakers in the Rooftop Terrace + area to 50%. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Set outdoor speakers volume to 50% + context: + id: 01J6SQSHN3EVC7WJHENTWJ76BX + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:06:24. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Outdoor Speakers\n domain: media_player\n state:\ + \ playing\n areas: Rooftop Terrace\n attributes:\n volume_level: 0.9\n\ + \ device_class: speaker\n" + messages: + - role: user + content: Set outdoor speakers volume to 50% + - role: assistant + content: + - type: tool_use + id: toolu_018YbUN8JRNaTExPrrWGemHz + name: HassSetVolume + input: + area: Rooftop Terrace + domain: + - media_player + name: Outdoor Speakers + volume_level: 50 + - role: user + content: + - type: tool_result + tool_use_id: toolu_018YbUN8JRNaTExPrrWGemHz + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Rooftop Terrace", "type": "area", "id": "rooftop_terrace"}, + {"name": "Outdoor Speakers", "type": "entity", "id": "media_player.outdoor_speakers"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I have adjusted the volume of the Outdoor Speakers in the Rooftop + Terrace area to 50%. + - event_type: tool_call + data: + tool_name: HassSetVolume + tool_args: + area: Rooftop Terrace + domain: + - media_player + name: Outdoor Speakers + volume_level: 50 diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-skip_song_on_outdoor_speakers.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-skip_song_on_outdoor_speakers.yaml new file mode 100644 index 00000000..ce77c171 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-skip_song_on_outdoor_speakers.yaml @@ -0,0 +1,77 @@ +--- +uuid: a2187c11-189c-4c30-ab58-4bbac192e6ef +task_id: home7_dk_media_player_media_player-skip_song_on_outdoor_speakers +category: media-player +task: + input_text: Skip song on outdoor speakers + expect_changes: + media_player.outdoor_speakers: + state: null + attributes: + media_track: 2 +response: I've skipped to the next song on the Outdoor Speakers in the Rooftop Terrace + area. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Skip song on outdoor speakers + context: + id: 01J6SQV5MWJH1TFGKW53GSZ11N + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:07:17. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Outdoor Speakers\n domain: media_player\n state:\ + \ playing\n areas: Rooftop Terrace\n attributes:\n volume_level: 0.5\n\ + \ device_class: speaker\n" + messages: + - role: user + content: Skip song on outdoor speakers + - role: assistant + content: + - type: text + text: Okay, let me do that for you. + - type: tool_use + id: toolu_013KYgEkY5PZe2jy2uMGHAsR + name: HassMediaNext + input: + area: Rooftop Terrace + domain: + - media_player + name: Outdoor Speakers + - role: user + content: + - type: tool_result + tool_use_id: toolu_013KYgEkY5PZe2jy2uMGHAsR + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Rooftop Terrace", "type": "area", "id": "rooftop_terrace"}, + {"name": "Outdoor Speakers", "type": "entity", "id": "media_player.outdoor_speakers"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I've skipped to the next song on the Outdoor Speakers in the Rooftop + Terrace area. + - event_type: tool_call + data: + tool_name: HassMediaNext + tool_args: + area: Rooftop Terrace + domain: + - media_player + name: Outdoor Speakers diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-skip_to_the_next_track_on_the_outdoor_speakers.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-skip_to_the_next_track_on_the_outdoor_speakers.yaml new file mode 100644 index 00000000..44bcd062 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-skip_to_the_next_track_on_the_outdoor_speakers.yaml @@ -0,0 +1,75 @@ +--- +uuid: b83e26d0-05ac-4ea8-9775-9674eb9bc813 +task_id: home7_dk_media_player_media_player-skip_to_the_next_track_on_the_outdoor_speakers +category: media-player +task: + input_text: Skip to the next track on the outdoor speakers + expect_changes: + media_player.outdoor_speakers: + state: null + attributes: + media_track: 2 +response: I've sent the command to skip to the next track on the Outdoor Speakers + in the Rooftop Terrace area. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Skip to the next track on the outdoor speakers + context: + id: 01J6SNTH54BK1K672QSD30GPGD + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:31:59. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Outdoor Speakers\n domain: media_player\n state:\ + \ playing\n areas: Rooftop Terrace\n attributes:\n volume_level: 0.5\n\ + \ device_class: speaker\n" + messages: + - role: user + content: Skip to the next track on the outdoor speakers + - role: assistant + content: + - type: tool_use + id: toolu_01NbYt97vZyuWmhjqmbNMXFb + name: HassMediaNext + input: + area: Rooftop Terrace + domain: + - media_player + name: Outdoor Speakers + - role: user + content: + - type: tool_result + tool_use_id: toolu_01NbYt97vZyuWmhjqmbNMXFb + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Rooftop Terrace", "type": "area", "id": "rooftop_terrace"}, + {"name": "Outdoor Speakers", "type": "entity", "id": "media_player.outdoor_speakers"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I've sent the command to skip to the next track on the Outdoor Speakers + in the Rooftop Terrace area. + - event_type: tool_call + data: + tool_name: HassMediaNext + tool_args: + area: Rooftop Terrace + domain: + - media_player + name: Outdoor Speakers diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-stop_the_outdoor_speakers.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-stop_the_outdoor_speakers.yaml new file mode 100644 index 00000000..ec61b920 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-stop_the_outdoor_speakers.yaml @@ -0,0 +1,76 @@ +--- +uuid: 77acd87e-ba3e-4a6d-9034-8f4a5d8919db +task_id: home7_dk_media_player_media_player-stop_the_outdoor_speakers +category: media-player +task: + input_text: Stop the Outdoor Speakers + expect_changes: + media_player.outdoor_speakers: + state: paused + attributes: null +response: The Outdoor Speakers have been turned off. +context: + unexpected_states: + media_player.outdoor_speakers: + expected: + state: paused + media_track: 0 + volume_level: 0.5 + got: + state: 'off' + media_track: null + volume_level: null + conversation_trace: + - event_type: async_process + data: + text: Stop the Outdoor Speakers + context: + id: 01J6SRBACNEVC7WJHENTWJ76BX + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 09:16:07. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Outdoor Speakers\n domain: media_player\n state:\ + \ playing\n areas: Rooftop Terrace\n attributes:\n volume_level: 0.5\n\ + \ device_class: speaker\n" + messages: + - role: user + content: Stop the Outdoor Speakers + - role: assistant + content: + - type: tool_use + id: toolu_01V29669U1bBrkQdbeBAAixT + name: HassTurnOff + input: + name: Outdoor Speakers + domain: media_player + - role: user + content: + - type: tool_result + tool_use_id: toolu_01V29669U1bBrkQdbeBAAixT + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Outdoor Speakers", "type": "entity", "id": "media_player.outdoor_speakers"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: The Outdoor Speakers have been turned off. + - event_type: tool_call + data: + tool_name: HassTurnOff + tool_args: + name: Outdoor Speakers + domain: media_player diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-turn_the_volume_down_to_50.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-turn_the_volume_down_to_50.yaml new file mode 100644 index 00000000..301285c1 --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-turn_the_volume_down_to_50.yaml @@ -0,0 +1,75 @@ +--- +uuid: a55b0a5a-7713-4c43-baf3-d97301b4bbdb +task_id: home7_dk_media_player_media_player-turn_the_volume_down_to_50 +category: media-player +task: + input_text: Turn the volume down to 50% + expect_changes: + media_player.outdoor_speakers: + state: null + attributes: + volume_level: 0.5 +response: Turned down the volume of the Outdoor Speakers in the Rooftop Terrace area + to 50%. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Turn the volume down to 50% + context: + id: 01J6SNWAX7EHCCVH61NKPJBY3A + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:32:58. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Outdoor Speakers\n domain: media_player\n state:\ + \ playing\n areas: Rooftop Terrace\n attributes:\n volume_level: 0.9\n\ + \ device_class: speaker\n" + messages: + - role: user + content: Turn the volume down to 50% + - role: assistant + content: + - type: tool_use + id: toolu_01EWEuVWJbbax2knJ8v5Qi55 + name: HassSetVolume + input: + area: Rooftop Terrace + device_class: + - speaker + volume_level: 50 + - role: user + content: + - type: tool_result + tool_use_id: toolu_01EWEuVWJbbax2knJ8v5Qi55 + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Rooftop Terrace", "type": "area", "id": "rooftop_terrace"}, + {"name": "Outdoor Speakers", "type": "entity", "id": "media_player.outdoor_speakers"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: Turned down the volume of the Outdoor Speakers in the Rooftop Terrace + area to 50%. + - event_type: tool_call + data: + tool_name: HassSetVolume + tool_args: + area: Rooftop Terrace + device_class: + - speaker + volume_level: 50 diff --git a/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-unpause_the_rooftop_terrace_music.yaml b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-unpause_the_rooftop_terrace_music.yaml new file mode 100644 index 00000000..d1bf644d --- /dev/null +++ b/reports/assist-mini/2024.9.0b2/claude-3-haiku/home7_dk_media_player_media_player-unpause_the_rooftop_terrace_music.yaml @@ -0,0 +1,70 @@ +--- +uuid: a8a4439a-fdf4-49ba-a317-fe06284c9f34 +task_id: home7_dk_media_player_media_player-unpause_the_rooftop_terrace_music +category: media-player +task: + input_text: Unpause the Rooftop Terrace music + expect_changes: + media_player.outdoor_speakers: + state: playing + attributes: null +response: I have resumed playback on the Rooftop Terrace speakers. +context: + unexpected_states: {} + conversation_trace: + - event_type: async_process + data: + text: Unpause the Rooftop Terrace music + context: + id: 01J6SNR8SB1B5TQRP1CB9W4WJK + parent_id: null + user_id: null + conversation_id: null + device_id: null + language: en + agent_id: conversation.mock_title + - event_type: agent_detail + data: + system: "Current time is 08:30:45. Today's date is 2024-09-02.\nYou are a voice\ + \ assistant for Home Assistant.\nAnswer questions about the world truthfully.\n\ + Answer in plain text. Keep it simple and to the point.\nWhen controlling Home\ + \ Assistant always call the intent tools. Use HassTurnOn to lock and HassTurnOff\ + \ to unlock a lock. When controlling a device, prefer passing just name and\ + \ domain. When controlling an area, prefer passing just area name and domain.\n\ + When a user asks to turn on all devices of a specific type, ask user to specify\ + \ an area, unless there is only one device of that type.\nThis device is not\ + \ able to start timers.\nAn overview of the areas and the devices in this\ + \ smart home:\n- names: Outdoor Speakers\n domain: media_player\n state:\ + \ paused\n areas: Rooftop Terrace\n attributes:\n volume_level: 0.5\n\ + \ device_class: speaker\n" + messages: + - role: user + content: Unpause the Rooftop Terrace music + - role: assistant + content: + - type: tool_use + id: toolu_01Jpt9T182TwmHR6mSAxwQHL + name: HassMediaUnpause + input: + area: Rooftop Terrace + domain: + - media_player + - role: user + content: + - type: tool_result + tool_use_id: toolu_01Jpt9T182TwmHR6mSAxwQHL + content: '{"speech": {}, "response_type": "action_done", "data": {"targets": + [], "success": [{"name": "Rooftop Terrace", "type": "area", "id": "rooftop_terrace"}, + {"name": "Outdoor Speakers", "type": "entity", "id": "media_player.outdoor_speakers"}], + "failed": []}}' + - role: assistant + content: + - type: text + text: I have resumed playback on the Rooftop Terrace speakers. + - event_type: tool_call + data: + tool_name: HassMediaUnpause + tool_args: + area: Rooftop Terrace + domain: + - media_player diff --git a/reports/assist-mini/2024.9.0b2/reports.yaml b/reports/assist-mini/2024.9.0b2/reports.yaml index 8ba05832..6ede3ca7 100644 --- a/reports/assist-mini/2024.9.0b2/reports.yaml +++ b/reports/assist-mini/2024.9.0b2/reports.yaml @@ -3,4 +3,8 @@ good_percent: 95.9% good: 47 total: 49 +- model_id: claude-3-haiku + good_percent: 98.0% + good: 48 + total: 49