diff --git a/datasets/anomaly/README.md b/datasets/anomaly/README.md index 5ffb5888..f0103730 100644 --- a/datasets/anomaly/README.md +++ b/datasets/anomaly/README.md @@ -1,4 +1,4 @@ -## Anomolies +## Anomalies -This dataset is used for classifying area summaries as anomolies vs classifying +This dataset is used for classifying area summaries as anomalies vs classifying as operating normally diff --git a/datasets/assist-mini/README.md b/datasets/assist-mini/README.md index 2e67922b..5b97e66e 100644 --- a/datasets/assist-mini/README.md +++ b/datasets/assist-mini/README.md @@ -1,13 +1,13 @@ # Mini Evaluation dataset for Home Assistant assist actions -This is a variation on the `assist` dataset that tests far fewer entites, devices, +This is a variation on the `assist` dataset that tests far fewer entities, devices, and areas at once. This is designed given poor performance of local models on the `assist` baseline to help with even smaller tasks to get right. This is a dataset for the Home Assistant LLM API ([blog post](https://developers.home-assistant.io/blog/2024/05/20/llm-api/)). See the `home-assistant-datasets assist` command for more details on how to -run evalations. +run evaluations. See [../assist/README.md](../assist/README.md) for details on how the dataset is configured including the inventory fixtures, and the eval tasks. diff --git a/datasets/assist-mini/home1-us-vacuum/vacuum.yaml b/datasets/assist-mini/home1-us-vacuum/vacuum.yaml index 5fac7321..c1bf1f4f 100644 --- a/datasets/assist-mini/home1-us-vacuum/vacuum.yaml +++ b/datasets/assist-mini/home1-us-vacuum/vacuum.yaml @@ -2,7 +2,7 @@ category: vacuum tests: - sentences: - - Start vaccuum in the living room + - Start vacuum in the living room - Start Roborock Downstairs vacuum setup: vacuum.roborock_downstairs: @@ -12,7 +12,7 @@ tests: state: cleaning - sentences: - Return Roborock Downstairs to base - - Return vaccuum in the living room to base + - Return vacuum in the living room to base setup: vacuum.roborock_downstairs: state: cleaning diff --git a/datasets/assist/README.md b/datasets/assist/README.md index e207681e..050b72c1 100644 --- a/datasets/assist/README.md +++ b/datasets/assist/README.md @@ -3,7 +3,7 @@ This is a dataset for the Home Assistant LLM API ([blog post](https://developers.home-assistant.io/blog/2024/05/20/llm-api/)). See the `home-assistant-datasets assist` command for more details on how to -run evalations. +run evaluations. ## Dataset details @@ -116,13 +116,14 @@ $ pip3 install -e /workspaces/core ``` You will need the [synthetic-home custom component](https://github.com/allenporter/synthetic-home) -and you can either install it in a separate direcotry like this: +and you can either install it in a separate directory like this: + ```bash $ export PYTHONPATH="${PYTHONPATH}:/workspaces/home-assistant-synthetic-home/" ``` Or using a `custom_components` directory in the local directory if you have multiple -custom components you wnat to evaluate: +custom components you want to evaluate: ```bash $ export PYTHONPATH="${PYTHONPATH}:${PWD}" diff --git a/datasets/intents/README.md b/datasets/intents/README.md index 0f72ecec..4df48581 100644 --- a/datasets/intents/README.md +++ b/datasets/intents/README.md @@ -3,7 +3,7 @@ This is a dataset for the Home Assistant LLM API ([blog post](https://developers.home-assistant.io/blog/2024/05/20/llm-api/)) imported from the home assistant intents repo. See the `home-assistant-datasets assist` command for more details on how to -run evalations. +run evaluations. ## Dataset details diff --git a/generation/README.md b/generation/README.md index dbf602e9..e49181a1 100644 --- a/generation/README.md +++ b/generation/README.md @@ -3,6 +3,7 @@ This directory contains notebooks and seeds for generating the datasets. The phases of generating data are currently: + - Generate a description of a Home ([notebook](homes.ipynb)) - Generate the Areas of the Home useful for automation ([notebook](areas.ipynb)) - Generate the devices in each area ([notebook](devices.ipynb)) @@ -20,17 +21,17 @@ The data is generated in a format that can be consumed by https://github.com/all The next phase needed is to generate useful state data for triggering those actions. -### Notebook Development enviroment +### Notebook Development environment This is an example of how to prepare the python venv for use with the jupyet notebooks: 1. Create the virtual environment - ```bash - $ python3 -m venv venv - $ source venv/bin/activate - $ pip install --upgrade pip - $ pip3 install -r requirements_dev.txt - ``` + ```bash + $ python3 -m venv venv + $ source venv/bin/activate + $ pip install --upgrade pip + $ pip3 install -r requirements_dev.txt + ``` 1. Open the notebook and select the venv interpreter and kernel diff --git a/generation/archive/area-summary.ipynb b/generation/archive/area-summary.ipynb index 33ea6190..7d29c74f 100644 --- a/generation/archive/area-summary.ipynb +++ b/generation/archive/area-summary.ipynb @@ -172,7 +172,7 @@ "which will be fed as input into other agents. The user will feed in details about\n", "areas and devices in the home, and you will respond with a summary of the status of the area.\n", "\n", - "Your summaries are succint, and do not mention boring details or things that seem\n", + "Your summaries are succinct, and do not mention boring details or things that seem\n", "very mundane or minor. A one sentence summary is best.\n", "\n", "\n", @@ -240,7 +240,7 @@ " if unit_of_measurement := attributes.get('unit_of_measurement'):\n", " state_value = f\"{state_value} {unit_of_measurement}\"\n", "\n", - " data.append(f\" {friendly_name} ({entity_name}): {state_value}\") \n", + " data.append(f\" {friendly_name} ({entity_name}): {state_value}\")\n", " return data\n", "\n", "for area in home.areas:\n", @@ -283,7 +283,7 @@ "source": [ "OVERALL_PROMPT = f\"\"\"\n", "You are an agent running in Home Assistant that is the overall house summary\n", - "agent. Other agents have summarized invidivual areas, and you are responsible\n", + "agent. Other agents have summarized individual areas, and you are responsible\n", "for picking out the important details, ignoring the mundane. You write succinct,\n", "1 sentence summaries of the entire house that can be displayed on a dashboard.\n", "You don't need to comment on every area, and can omit any that are mundane.\n", @@ -323,7 +323,7 @@ "- Attic: The attic temperature is high.\n", "- Front Door The front door is being used and the porch is dimmed.\n", "\n", - "Summary: \n", + "Summary:\n", "\"\"\"\n", "\n", "response = model.complete(OVERALL_PROMPT, message)\n", diff --git a/generation/devices.ipynb b/generation/devices.ipynb index 066a2af6..380eee3a 100644 --- a/generation/devices.ipynb +++ b/generation/devices.ipynb @@ -170,7 +170,7 @@ "- Balcony\n", "\n", "Output:\n", - "# thought: I will genearate a set of devices commonly found in a New York City Apartment\n", + "# thought: I will generate a set of devices commonly found in a New York City Apartment\n", "# thought: Given the compact nature of the living space, the devices may focus on essential functionalities and convenience for the resident\n", "Living Room:\n", "- name: Lamp\n", @@ -310,7 +310,7 @@ "You are an expert in smart home automation and are generating data used to\n", "evaluate the performance of a smart home system Home Assistant on tasks like\n", "summarization, performing actions, or being an independent agent managing\n", - "automations and maintenace tasks.\n", + "automations and maintenance tasks.\n", "\n", "You use your knowledge about the world to generate details about homes that\n", "can be used for synthetic smart home automation data. For example, an apartment\n", @@ -318,7 +318,7 @@ "smart lock and camera, and all houses may have a smart light or weather feed\n", "air quality, or a smart speaker or television. The needs of a home owner\n", "may vary if they are a single person or family, or where in the world they\n", - "live. For example, a high rise aparment probably does not have a backyard.\n", + "live. For example, a high rise apartment probably does not have a backyard.\n", "\n", "Do not be cliche. Do not assume everyone has a smart watch. Don't assume\n", "every room has a smart light just because it is listed, but of course many will.\n", diff --git a/generation/seeds/devices.yaml b/generation/seeds/devices.yaml index f26e0d2b..8d241d88 100644 --- a/generation/seeds/devices.yaml +++ b/generation/seeds/devices.yaml @@ -2,11 +2,11 @@ home: name: Coastal Bungalow thoughts: - - The bungalow in a seaside village may be a vacation home or a tranquil retreat. - - The cozy living room with a fireplace hints at a cozy atmosphere, and there may - be smart climate control for comfort. - - The outdoor shower is convenient for beach days, so there may be smart water heating - systems. + - The bungalow in a seaside village may be a vacation home or a tranquil retreat. + - The cozy living room with a fireplace hints at a cozy atmosphere, and there may + be smart climate control for comfort. + - The outdoor shower is convenient for beach days, so there may be smart water heating + systems. desc: Bungalow in a seaside village in Norway areas: - Living Room @@ -20,7 +20,7 @@ devices: - name: Living Room device_type: light-dimmable device_info: - model: Enbrighten Z-Wave + model: Enbrighten Z-Wave manufacturer: GE sw_version: 2.3.29-7 - name: Thermostat @@ -42,7 +42,7 @@ devices: - name: Port light device_type: light device_info: - model: Enbrighten Z-Wave + model: Enbrighten Z-Wave manufacturer: GE sw_version: 2.3.29-7 Outdoor Shower: @@ -57,9 +57,9 @@ devices: home: name: Home2 thoughts: - - The apartment with city views may cater to a single person or a couple. - - The shared rooftop terrace could have restrictions on device installations. - - In-unit laundry suggests convenience, so there may be a smart washer and dryer. + - The apartment with city views may cater to a single person or a couple. + - The shared rooftop terrace could have restrictions on device installations. + - In-unit laundry suggests convenience, so there may be a smart washer and dryer. desc: Apartment in urban area in New York City, USA areas: - Living Room @@ -67,7 +67,7 @@ home: - Bedroom - Balcony devices-thoughts: - - I will genearate a set of devices commonly found in a New York City Apartment + - I will generate a set of devices commonly found in a New York City Apartment - Given the compact nature of the living space, the devices may focus on essential functionalities and convenience for the resident devices: Living Room: @@ -113,12 +113,12 @@ devices: home: name: Loft Industrial thoughts: - - The industrial loft with an open space and high ceilings may have a modern and - minimalist design. - - The panoramic windows offer plenty of natural light, ideal for a smart lighting - system. - - The rooftop terrace could be a great spot for outdoor entertainment, so there - may be smart speakers or a sound system. + - The industrial loft with an open space and high ceilings may have a modern and + minimalist design. + - The panoramic windows offer plenty of natural light, ideal for a smart lighting + system. + - The rooftop terrace could be a great spot for outdoor entertainment, so there + may be smart speakers or a sound system. desc: Industrial loft in Barcelona, Spain areas: - Living Space diff --git a/home_assistant_datasets/tool/archive_evals/anomaly/output/gemma-3-shot.yaml b/home_assistant_datasets/tool/archive_evals/anomaly/output/gemma-3-shot.yaml index 01ec8c25..fe7a8493 100644 --- a/home_assistant_datasets/tool/archive_evals/anomaly/output/gemma-3-shot.yaml +++ b/home_assistant_datasets/tool/archive_evals/anomaly/output/gemma-3-shot.yaml @@ -313,7 +313,7 @@ response: anomaly uuid: 1e4a7577-5d18-4dc5-b292-92ac333efa93 task_id: d23ea7aac86432d2b3137e86965178ff96336f895742bdfa6dc627f67073dde5-anomaly task: - text: The smart plugs connected to the refridgerator is currently off. + text: The smart plugs connected to the refrigerator is currently off. label: anomaly response: anomaly --- diff --git a/home_assistant_datasets/tool/archive_evals/anomaly/output/gpt-3.5-10-shot.yaml b/home_assistant_datasets/tool/archive_evals/anomaly/output/gpt-3.5-10-shot.yaml index df781196..3a6127b8 100644 --- a/home_assistant_datasets/tool/archive_evals/anomaly/output/gpt-3.5-10-shot.yaml +++ b/home_assistant_datasets/tool/archive_evals/anomaly/output/gpt-3.5-10-shot.yaml @@ -2,7 +2,7 @@ uuid: 1845a3d9-359e-4043-bbde-593687853c79 task_id: d23ea7aac86432d2b3137e86965178ff96336f895742bdfa6dc627f67073dde5-anomaly task: - text: The smart plugs connected to the refridgerator is currently off. + text: The smart plugs connected to the refrigerator is currently off. label: anomaly response: anomaly --- diff --git a/home_assistant_datasets/tool/archive_evals/anomaly/output/llama3-10-shot.yaml b/home_assistant_datasets/tool/archive_evals/anomaly/output/llama3-10-shot.yaml index 8be6d70f..dea1638d 100644 --- a/home_assistant_datasets/tool/archive_evals/anomaly/output/llama3-10-shot.yaml +++ b/home_assistant_datasets/tool/archive_evals/anomaly/output/llama3-10-shot.yaml @@ -273,7 +273,7 @@ response: normal uuid: eb56c6db-7127-491c-aa9c-eafc23e830e4 task_id: d23ea7aac86432d2b3137e86965178ff96336f895742bdfa6dc627f67073dde5-anomaly task: - text: The smart plugs connected to the refridgerator is currently off. + text: The smart plugs connected to the refrigerator is currently off. label: anomaly response: anomaly --- diff --git a/home_assistant_datasets/tool/archive_evals/anomaly/output/llama3-zero-shot.yaml b/home_assistant_datasets/tool/archive_evals/anomaly/output/llama3-zero-shot.yaml index dd165f47..845d9bc4 100644 --- a/home_assistant_datasets/tool/archive_evals/anomaly/output/llama3-zero-shot.yaml +++ b/home_assistant_datasets/tool/archive_evals/anomaly/output/llama3-zero-shot.yaml @@ -48,7 +48,7 @@ response: normal uuid: 383a566a-f7d6-4694-976c-9a0717432ed2 task_id: d23ea7aac86432d2b3137e86965178ff96336f895742bdfa6dc627f67073dde5-anomaly task: - text: The smart plugs connected to the refridgerator is currently off. + text: The smart plugs connected to the refrigerator is currently off. label: anomaly response: anomaly --- diff --git a/home_assistant_datasets/tool/archive_evals/anomaly/output/mistral-7b-instruct-10-shot.yaml b/home_assistant_datasets/tool/archive_evals/anomaly/output/mistral-7b-instruct-10-shot.yaml index 659b5235..9e6111a5 100644 --- a/home_assistant_datasets/tool/archive_evals/anomaly/output/mistral-7b-instruct-10-shot.yaml +++ b/home_assistant_datasets/tool/archive_evals/anomaly/output/mistral-7b-instruct-10-shot.yaml @@ -540,7 +540,7 @@ response: normal uuid: 5e37847c-31ab-4de2-85c3-1ae45c2b2dbe task_id: d23ea7aac86432d2b3137e86965178ff96336f895742bdfa6dc627f67073dde5-anomaly task: - text: The smart plugs connected to the refridgerator is currently off. + text: The smart plugs connected to the refrigerator is currently off. label: anomaly response: normal --- diff --git a/home_assistant_datasets/tool/archive_evals/anomaly/output/mistral-7b-instruct-zero-shot.yaml b/home_assistant_datasets/tool/archive_evals/anomaly/output/mistral-7b-instruct-zero-shot.yaml index 442440a2..a1cf98fe 100644 --- a/home_assistant_datasets/tool/archive_evals/anomaly/output/mistral-7b-instruct-zero-shot.yaml +++ b/home_assistant_datasets/tool/archive_evals/anomaly/output/mistral-7b-instruct-zero-shot.yaml @@ -46,7 +46,7 @@ response: anomaly uuid: 70a75c30-e7c6-4a9b-b5a5-a0c78afa9b22 task_id: d23ea7aac86432d2b3137e86965178ff96336f895742bdfa6dc627f67073dde5-anomaly task: - text: The smart plugs connected to the refridgerator is currently off. + text: The smart plugs connected to the refrigerator is currently off. label: anomaly response: anomaly --- diff --git a/home_assistant_datasets/tool/archive_evals/anomaly/test_anomaly_eval.py b/home_assistant_datasets/tool/archive_evals/anomaly/test_anomaly_eval.py index 771a191a..03212c23 100644 --- a/home_assistant_datasets/tool/archive_evals/anomaly/test_anomaly_eval.py +++ b/home_assistant_datasets/tool/archive_evals/anomaly/test_anomaly_eval.py @@ -58,7 +58,7 @@ ], ) def model_id_fixture(request: pytest.FixtureRequest) -> str: - """Fiture that defines which model is being evaluated.""" + """Fixture that defines which model is being evaluated.""" return request.param diff --git a/home_assistant_datasets/tool/archive_evals/archive/area_summary/eval.py b/home_assistant_datasets/tool/archive_evals/archive/area_summary/eval.py index b1499614..d48871ed 100644 --- a/home_assistant_datasets/tool/archive_evals/archive/area_summary/eval.py +++ b/home_assistant_datasets/tool/archive_evals/archive/area_summary/eval.py @@ -17,7 +17,7 @@ AREA_SUMMARY_PROMPT = """ -Please summarize the status of an area of the home. Your summaries are succint, +Please summarize the status of an area of the home. Your summaries are succinct, and do not mention boring details or things that seem very mundane or minor. A one sentence summary is best. @@ -52,7 +52,6 @@ """ - def make_prompt(area_name: str) -> str: """Create a prompt for the agent to summarize the area.""" return AREA_PROMPT.format( @@ -81,7 +80,6 @@ async def async_process(self, hass: HomeAssistant, text: str) -> str: return response["speech"]["plain"]["speech"] - async def async_run_eval(hass: HomeAssistant, config: dict[str, Any]) -> None: """Run the evaluation.""" @@ -89,17 +87,25 @@ async def async_run_eval(hass: HomeAssistant, config: dict[str, Any]) -> None: os.makedirs(eval_dir, exist_ok=True) # Find the conversation agent id to evaluate based on the domain in the config file - config_entry_ids = [ entry.entry_id for entry in hass.config_entries.async_entries() if entry.domain in config["conversation_agent_domain"] ] + config_entry_ids = [ + entry.entry_id + for entry in hass.config_entries.async_entries() + if entry.domain in config["conversation_agent_domain"] + ] if len(config_entry_ids) > 1: - raise ValueError("Found more than one config entry for domain {config['convesation_agent_domain']}") + raise ValueError( + "Found more than one config entry for domain {config['convesation_agent_domain']}" + ) if not config_entry_ids: - raise ValueError("Found no config entry for domain {config['convesation_agent_domain']}") + raise ValueError( + "Found no config entry for domain {config['convesation_agent_domain']}" + ) conversation_agent_id = config_entry_ids[0] agent = ConversationAgent(conversation_agent_id) area_registry = ar.async_get(hass) - area_names = [ area.name for area in area_registry.async_list_areas() ] + area_names = [area.name for area in area_registry.async_list_areas()] _LOGGER.info("Loaded %s areas to evaluate", len(area_names)) diff --git a/home_assistant_datasets/tool/archive_evals/archive/eval/README.md b/home_assistant_datasets/tool/archive_evals/archive/eval/README.md index d286e5e0..7ae78169 100644 --- a/home_assistant_datasets/tool/archive_evals/archive/eval/README.md +++ b/home_assistant_datasets/tool/archive_evals/archive/eval/README.md @@ -53,6 +53,7 @@ $ cat evals/area_summary/out/*.yaml | head -11 ``` Each area has its own file that contains the instructions and the LLM response in the `response` field: + ```yaml area: Backyard response: In the backyard, there are Deck Lights which are outdoor smart string lights @@ -69,7 +70,6 @@ response: In Bedroom 2, you have a warm glow bulb and a climate sensor (smart te 1. Rate the results (not yet managed) - ### Seed the devices in the Home The above example will run Home Assistant, will setup the home, complete onboarding, and create the synthetic devices @@ -78,15 +78,18 @@ using the [Synthetic Home integration](https://github.com/allenporter/home-assis You can run the evaluation with --no-delete_tmpdir and inspect the state of the created Home. ```bash -# Get the temprary dir from the script.eval +# Get the temporary dir from the script.eval $ OUTPUT_DIR=/tmp/homeassistant-evalrqdy3axl ``` Verify synthetic areas were created properly: + ```bash $ cat "${OUTPUT_DIR}/config/.storage/core.area_registry" | jq '.data.areas[].name' | head -5 ``` + Will output: + ```json "Kitchen" "Living Room" @@ -96,10 +99,13 @@ Will output: ``` Verify synthetic devices were created properly: + ```bash $ cat "${OUTPUT_DIR}/config/.storage/core.device_registry" | jq '.data.devices[].name' | head -5 ``` + Will output: + ```json "Kitchen Light" "Refrigerator" @@ -111,10 +117,10 @@ Will output: The devices currently have a default state, but can be updated with [Restorable Attributes](https://github.com/allenporter/home-assistant-synthetic-home/?tab=readme-ov-file#restorable-attributes-using-service-calls) configured by the [Synthetic Home integration](https://github.com/allenporter/home-assistant-synthetic-home/). - ### Human Evaluation In the future we want to: + - Record the score (1, 2, 3) - Gather statistics on the results - Support many more homes and devices at once diff --git a/home_assistant_datasets/tool/archive_evals/archive/eval/__init__.py b/home_assistant_datasets/tool/archive_evals/archive/eval/__init__.py index 8c0bddfe..85d8b7de 100644 --- a/home_assistant_datasets/tool/archive_evals/archive/eval/__init__.py +++ b/home_assistant_datasets/tool/archive_evals/archive/eval/__init__.py @@ -1 +1 @@ -"""Tool for evaluating code using a Home Assitant runner.""" +"""Tool for evaluating code using a Home Assistant runner.""" diff --git a/home_assistant_datasets/tool/archive_evals/archive/eval/config.py b/home_assistant_datasets/tool/archive_evals/archive/eval/config.py index 1a92a971..042bb1e9 100644 --- a/home_assistant_datasets/tool/archive_evals/archive/eval/config.py +++ b/home_assistant_datasets/tool/archive_evals/archive/eval/config.py @@ -23,7 +23,6 @@ _LOGGER = logging.getLogger(__name__) - CONF_COPY_CONFIG_FILES = "copy_config_files" CONF_CONFIG_ENTRIES = "config_entries" @@ -55,7 +54,7 @@ "analytics", "integration", ] - } + }, } @@ -67,14 +66,14 @@ async def async_setup(hass: HomeAssistant, config: dict[str, Any]) -> None: # await config.async_create_default_config(hass) config_dir = pathlib.Path(hass.config.config_dir) - with (config_dir / CONFIURATION_YAML).open('w') as configuration_yaml_file: + with (config_dir / CONFIURATION_YAML).open("w") as configuration_yaml_file: configuration_yaml_file.write(yaml.dump(MINIMAL_CONFIG, sort_keys=False)) os.makedirs(config_dir / STORAGE_DIR, exist_ok=True) - with (config_dir / ONBOARDING_FILE).open('w') as onboarding_file: + with (config_dir / ONBOARDING_FILE).open("w") as onboarding_file: json.dump(ONBOARDING_CONFIG, onboarding_file) - # Copy eval specifc configuration files + # Copy eval specific configuration files for config_file in config.get(CONF_COPY_CONFIG_FILES, []): shutil.copy( config_file, @@ -84,7 +83,7 @@ async def async_setup(hass: HomeAssistant, config: dict[str, Any]) -> None: def create_config_entry(config_entry: dict[str, Any]) -> config_entries.ConfigEntry: """Create a ConfigEntry from a dict.""" - # Arbitarily assign a unique id based on the contents of the entry + # Arbitrarily assign a unique id based on the contents of the entry hash = hashlib.sha256() hash.update(str(config_entry).encode()) unique_id = hash.hexdigest() @@ -101,7 +100,9 @@ def create_config_entry(config_entry: dict[str, Any]) -> config_entries.ConfigEn ) -async def async_setup_config_entries(hass: HomeAssistant, config: dict[str, Any]) -> None: +async def async_setup_config_entries( + hass: HomeAssistant, config: dict[str, Any] +) -> None: """Perform setup of any config entries specified in the dictionary.""" for config_entry_dict in config.get(CONF_CONFIG_ENTRIES, []): @@ -110,5 +111,11 @@ async def async_setup_config_entries(hass: HomeAssistant, config: dict[str, Any] await hass.config_entries.async_add(config_entry) await hass.async_block_till_done() if config_entry.state != config_entries.ConfigEntryState.LOADED: - _LOGGER.warning("Config entry %s failed to load: %s", config_entry.domain, config_entry.state) - raise ValueError(f"Failed to load config entry for domain {config_entry.domain}") + _LOGGER.warning( + "Config entry %s failed to load: %s", + config_entry.domain, + config_entry.state, + ) + raise ValueError( + f"Failed to load config entry for domain {config_entry.domain}" + ) diff --git a/home_assistant_datasets/tool/archive_evals/archive/eval/runner.py b/home_assistant_datasets/tool/archive_evals/archive/eval/runner.py index 2f8bcecb..eaa6609d 100644 --- a/home_assistant_datasets/tool/archive_evals/archive/eval/runner.py +++ b/home_assistant_datasets/tool/archive_evals/archive/eval/runner.py @@ -50,7 +50,7 @@ class RuntimeConfig: async def async_setup_config_entries(hass: HomeAssistant) -> None: - """Initialize HomeAssistant configuration and existing config entires.""" + """Initialize HomeAssistant configuration and existing config entries.""" config_dict = await conf_util.async_hass_config_yaml(hass) await bootstrap.async_from_config_dict(config_dict, hass) diff --git a/home_assistant_datasets/tool/archive_evals/area_summary/test_area_summary_agent_eval.py b/home_assistant_datasets/tool/archive_evals/area_summary/test_area_summary_agent_eval.py index c032986b..18a1cbad 100644 --- a/home_assistant_datasets/tool/archive_evals/area_summary/test_area_summary_agent_eval.py +++ b/home_assistant_datasets/tool/archive_evals/area_summary/test_area_summary_agent_eval.py @@ -49,7 +49,7 @@ ], ) def model_id_fixture(request: pytest.FixtureRequest) -> str: - """Fiture that defines which model is being evaluated.""" + """Fixture that defines which model is being evaluated.""" return request.param @@ -109,6 +109,7 @@ def eval_output_file_fixture(model_id: str, synthetic_home_config: str) -> str: home_file = pathlib.Path(synthetic_home_config).name return pathlib.Path(f"{MODEL_EVAL_OUTPUT}/{model_id}/{home_file}") + @dataclass class DeviceState: """Information needed to set the synthetic state for an evaluation task.""" diff --git a/home_assistant_datasets/tool/archive_evals/area_summary/test_area_summary_eval.py b/home_assistant_datasets/tool/archive_evals/area_summary/test_area_summary_eval.py index 27a9936b..c4c9dfa5 100644 --- a/home_assistant_datasets/tool/archive_evals/area_summary/test_area_summary_eval.py +++ b/home_assistant_datasets/tool/archive_evals/area_summary/test_area_summary_eval.py @@ -121,7 +121,7 @@ def cleanup_response(response: str) -> str: ], ) def model_id_fixture(request: pytest.FixtureRequest) -> str: - """Fiture that defines which model is being evaluated.""" + """Fixture that defines which model is being evaluated.""" return request.param @@ -135,7 +135,7 @@ def model_id_fixture(request: pytest.FixtureRequest) -> str: ], ) def synthetic_home_config_fixture(request: pytest.FixtureRequest) -> str: - """Fiture that defines which model is being evaluated.""" + """Fixture that defines which model is being evaluated.""" return request.param