Added context compatibility checks for file_surfer.

microsoft · Feb 4, 2025 · 8c351cf · 8c351cf
1 parent 8e80a52
commit 8c351cf
Showing 1 changed file with 31 additions and 2 deletions.
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/file_surfer/_file_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/file_surfer/_file_surfer.py
@@ -9,7 +9,7 @@
     MultiModalMessage,
     TextMessage,
 )
-from autogen_core import CancellationToken, FunctionCall
+from autogen_core import CancellationToken, FunctionCall, Image
 from autogen_core.models import (
     AssistantMessage,
     ChatCompletionClient,
@@ -126,7 +126,7 @@ async def _generate_reply(self, cancellation_token: CancellationToken) -> Tuple[
         )
 
         create_result = await self._model_client.create(
-            messages=history + [context_message, task_message],
+            messages=self._make_compatible_context(history + [context_message, task_message]),
             tools=[
                 TOOL_OPEN_PATH,
                 TOOL_PAGE_DOWN,
@@ -172,3 +172,32 @@ async def _generate_reply(self, cancellation_token: CancellationToken) -> Tuple[
 
         final_response = "TERMINATE"
         return False, final_response
+
+
+    def _content_to_str(self, content: str | List[str | Image]) -> str:
+        """Convert the content to a string."""
+        if isinstance(content, str):
+            return content
+        else:
+            result: List[str] = []
+            for c in content:
+                if isinstance(c, str):
+                    result.append(c)
+                else:
+                    result.append("<image>")
+        return "\n".join(result)
+
+
+    def _make_compatible_context(self, context: List[LLMMessage]):
+        """Ensure context are compatible with the underlying model."""
+        if self._model_client.model_info["vision"]:
+            return context
+        else:
+            new_context: List[LLMMessage] = []
+            for message in context:
+                if isinstance(message, UserMessage):
+                    # Only user messages can conain images
+                    new_context.append(UserMessage(content=self._content_to_str(message.content), source=message.source))
+                else:
+                    new_context.append(message)
+            return new_context