Format

fastchat/serve/monitor/classify/category.py

@@ -139,22 +139,30 @@ def post_process(self, judgment):
         score = self.get_score(judgment=judgment)
         return {"math": bool(score == "yes") if score else False}
 
+
 class CategoryVisionHardPrompt(CategoryHardPrompt):
     def __init__(self):
         super().__init__()
         self.name_tag = "criteria_vision_v0.1"
-    
+
     def _convert_filepath_to_base64(self, filepath):
         with open(filepath, "rb") as image_file:
-            return base64.b64encode(image_file.read()).decode('utf-8')
+            return base64.b64encode(image_file.read()).decode("utf-8")
 
     def pre_process(self, prompt: str, image_list: list):
         # Prompt is a list where the first element is text and the second element is a list of image in base64 format
         conv = [{"role": "system", "content": self.sys_prompt}]
         single_turn_content_list = []
         single_turn_content_list.append({"type": "text", "text": prompt})
         for image_url in image_list:
-            single_turn_content_list.append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{self._convert_filepath_to_base64(image_url)}"}})
-
+            single_turn_content_list.append(
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/png;base64,{self._convert_filepath_to_base64(image_url)}"
+                    },
+                }
+            )
+
         conv.append({"role": "user", "content": single_turn_content_list})
-        return conv
+        return conv

fastchat/serve/monitor/classify/label.py

@@ -167,7 +167,8 @@ def find_required_tasks(row):
         )
     ]
 
-def aggregate_entire_conversation(conversation, images_dir):              
+
+def aggregate_entire_conversation(conversation, images_dir):
     final_text_content = ""
     final_image_list = []
 
@@ -186,12 +187,15 @@ def aggregate_entire_conversation(conversation, images_dir):
 
     return final_text_content, final_image_list
 
+
 def get_prompt_from_conversation(conversation):
     return conversation[0]
 
+
 def get_image_list_from_conversation(conversation):
     return conversation[1]
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--config", type=str, required=True)
@@ -276,9 +280,11 @@ def get_image_list_from_conversation(conversation):
     not_labeled["prompt"] = not_labeled.conversation_a.map(
         lambda convo: aggregate_entire_conversation(convo, config["images_dir"])
     )
-    
+
     if config["images_dir"]:
-        not_labeled["image_list"] = not_labeled.prompt.map(get_image_list_from_conversation)
+        not_labeled["image_list"] = not_labeled.prompt.map(
+            get_image_list_from_conversation
+        )
         not_labeled = not_labeled[not_labeled.image_list.map(len) > 0]
     not_labeled["prompt"] = not_labeled.prompt.map(get_prompt_from_conversation)
     not_labeled["prompt"] = not_labeled.prompt.map(lambda x: x[:12500])

fastchat/serve/monitor/elo_analysis.py

@@ -735,7 +735,10 @@ def pretty_print_elo_rating(rating):
         "long": filter_long_conv,
         "chinese": lambda x: x["language"] == "Chinese",
         "english": lambda x: x["language"] == "English",
-        "criteria_vision_v0.1": lambda x: sum(x["category_tag"]["criteria_vision_v0.1"].values()) >= 6,
+        "criteria_vision_v0.1": lambda x: sum(
+            x["category_tag"]["criteria_vision_v0.1"].values()
+        )
+        >= 6,
     }
     assert all(
         [cat in filter_func_map for cat in args.category]