From 557083a156c3dd67ac79e22b4202e9b69b6b00f4 Mon Sep 17 00:00:00 2001
From: Yan Shu <570533048@qq.com>
Date: Mon, 8 Jul 2024 16:56:50 +0800
Subject: [PATCH] Add files via upload

---
 .../mlvu/__pycache__/utils.cpython-310.pyc    | Bin 0 -> 3207 bytes
 lmms_eval/tasks/mlvu/mlvu.yaml                |  21 +++
 lmms_eval/tasks/mlvu/utils.py                 | 124 ++++++++++++++++++
 3 files changed, 145 insertions(+)
 create mode 100644 lmms_eval/tasks/mlvu/__pycache__/utils.cpython-310.pyc
 create mode 100644 lmms_eval/tasks/mlvu/mlvu.yaml
 create mode 100644 lmms_eval/tasks/mlvu/utils.py

diff --git a/lmms_eval/tasks/mlvu/__pycache__/utils.cpython-310.pyc b/lmms_eval/tasks/mlvu/__pycache__/utils.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1a9fda0f2b70ce7af4bbcc1868a64d7c8e67571d
GIT binary patch
literal 3207
zcmZ`*-E$ka5eIO`<B_6hS(fz?+sWCk9Wk*b+wH_Ns+(4IobgN>*_PaPf+llwI)Kzk
z_d(;1w8c4g+Hv~ShxR=mL-noyME?-H=E?tpXENz7j*{&pJ@NqT-eLjl{&pAH>}-?3
z6MyrM?k8<RzD45o#|7dy@aEf;5J3cuNkTcLIA$^9%=8wwOmA}=dMmCa4tI=<9oG|=
zyHoiFZy32IZ$e*-XOb3gC9`}sY4dh+il3Uw&G9)ScbcCz{X9Pdy%R4aXZa!}JtFG=
zrhG}V$+B=o<B0JU$@brxoRcd@l%JOuq$}TkPWRst&2vP|JZJy(xF@~I#UsKm^$5Q#
ze<m+Iryr5eXFp^73co5^^6HSPp}ahKS6)4W^F4|F1ShnKO`cnnr0z|U|LrO8@%Jgb
zvN2f|vy+azd_<3^Xs-})YK8Fk#GE+&1>x_Dx5T_S^9AMC#DX{roa<syEJ42}mc<J6
z>w@i&&bb$8?GDv5LUw~eT!<(vwt!yl$yBPKkbXYcO`<%HvefTJv4n5-!=Tvf*xLSm
zlo#6m2&QQFLB9ZzAl9`%q%Z(6C(e33sX7hL9&c;5xy{)hceu6nXj8L$+uVBiXp>tz
zJ9jzTd;srVUBCaafB(TJ_eP6r>wj5a3xjY^uI~+cy(sN<gHVndNjw~^9R^9PsTwsd
zb_Z#Qt7@X#A}E4f7XHE0K=pFnFtdFTsSuEJ@>npc!Fw0pd>fjA?9&M=tinEGV_E{j
zR>+uD+>(|IxD{1{F;y2zpi9=HJEU+Xbphiq0dZej=(CF7vA1}07zvsAX^_a#j2U}<
zu-V4wf+sSWdudj9@)=m~n%9pd?AwE*h6f#2vn<y(%4&N*i&EX*g|Gg}VRW+^qGs(F
z6}h(ZLzo0a(Xq58vQW*z$@Fv+sO&#Poded(JL~ygpyc}gAl;875QD=YU5}F_zb=PC
zY(~BrfiVyAb+q^Tporppt$(QJk@P{oUu6C;$_GLGKH3piYcrR&X^Yy_vF5*bz*4{r
z88pWanjL9l`c)VR3Z?{*19Yt=GPX)4>?Kmd7<*9zo?Y4iQfEs+ZXnuyGLSj?dvw0}
zAdL^bUNn?mu_wJ<;Cq=t;967D=BWMKSDX!R)ttUlQ!A+Ee3A(n`*~l6Q8x<xzRHsR
zF>v^7rDn=iZ^H~e(;dX|F$WE4^r`cxZSHlW6!KYd8@M^<Et|FI0vJSH1mZ_1L?jwh
z+=4+#Cgg~W!6YRa614<06%f~m$dU@S4DH{+P>8)9kOrezTQ6zn1!B9ybS*Db)Yr8r
z74jK?fadB7d;ugWDXPtyZJQl0!zNV_7Jdj=fL^NHSF$Ic{Tim^EofZo(k7jQz1vKo
ztKM*=1B{ax`Tz*n;WTPW_MCzvjZ?vS;Mf&X+9h&qaC7Y|dt4JvS&JZkI`z?in;txy
z5U(3%gcQJQ!wbA9&5IxnrI&S$hY^(CG{#S4mCPY!<s06g@tmI<{<bUzDt%31;9*X~
z+@ub@r%|!z9mvBCFOguf(8ECTR-+UaiE=N*P&DzH06PyeB|FzVSb88&W*EvWrO^7i
zfuZ6Thka?lqMabkpF-NR^~^C|{Ya<|%Ff_E{YpT+2OPd&-~)3*iF@>`HL0<B$Eg5Y
zpPo~_i!<C~9hJ%k8U^xPSp3k>Q`IU9WsV*;wz>~0a;&+o>oS)uQkSW9Am6}e1HZ#-
z@cA*EG9qJI5U5v;F%#fz!M<Xc9q*7Q;AP-@;B^bV4G0B3_}Z=!^!ROJkvo78!*h@a
za&^<XE98Jx-=EAG?H`P7ND4K0osv%I5o%uejE(DK7f@7($aqxNN>?~5<Te@CN(aV(
zD;PUZepRrEB^sp-`M6m+f*#mvSIm^P1yVZaNQ%2|eQAr?uWP2h{`^NP%}rPOe=8`n
zLVK$FUq7k4ssq*PN3SGyAhH4Rr$+k*IBz2U1jT_DLrFly8;s?iGFX<v+dcFkOTF&L
zpT_!Th1pl=gxpXE6f21(y22eK1jM4ciOpqfZeVi_8-(QOoqM=^fN=ES{}|xy-G!S#
zFH@0(U>mUw?}b1~+grQYeMzs3&OR8zWg(8ehf;Mjm0;v<cn|{3$D2F9^B;fu@ZJt&
zr$Q#Vp21M|tGj`=qqLiK8tOWz<L$6o<?xn?f(qYGWgcFI8A|ZkA}c_}^jy4Mk;kXg
z^gz?$4})RR#97emL8il7&D7`|*3H+TvFk1*JlAp|WnY3GQXkS#(OQ)K^#&wDq+3wq
zR_QWRzl13ry4AU%XTmIw!MtFtTw#4vr=ig<ge62YRB3xZ&r*IBi>xWFYo@fW`6g{t
zwH5DGz8NYDC)crN$I-5`k+Si^qn2>!K$?Nekizt<Fb}IbHW;g#DXFb+cuUvPK?2p9
zrG0Y+8>oT8fL55Q)h|%g>B&N-LtWqN`g>U-)jV>W@>xGf#UPhTEg-|r`ZCq7Z|;4*
zubn>J9nwP6u5Kt9h&W1Ru3Z&8^)U(YMjmvf8R~|FO_=+mrjfQ$5)xw{^a)*w#aSTK
z8QjuYZ1B2Z3Z=<>CM6jozvjnR70kT@C43Oe4>1AfA41bIHLG$LR<C7v@z;X<)@rxt
cB3w_5Ls=8t$-X=P-E!Tvn=QzKjn@4C0A7ngVE_OC

literal 0
HcmV?d00001

diff --git a/lmms_eval/tasks/mlvu/mlvu.yaml b/lmms_eval/tasks/mlvu/mlvu.yaml
new file mode 100644
index 00000000..375da595
--- /dev/null
+++ b/lmms_eval/tasks/mlvu/mlvu.yaml
@@ -0,0 +1,21 @@
+dataset_path: sy1998/temp
+dataset_kwargs:
+  token: True
+  cache_dir: mlvu
+  video: True
+task: mlvu
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.mlvu_doc_to_visual
+doc_to_text: !function utils.mlvu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mlvu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+metric_list:
+  - metric: mlvu_percetion_score
+    aggregation: !function utils.mlvu_aggregate_results
+    higher_is_better: true
+
+
+
diff --git a/lmms_eval/tasks/mlvu/utils.py b/lmms_eval/tasks/mlvu/utils.py
new file mode 100644
index 00000000..476cf9dc
--- /dev/null
+++ b/lmms_eval/tasks/mlvu/utils.py
@@ -0,0 +1,124 @@
+from collections import defaultdict
+import os
+import datetime
+import json
+from lmms_eval.tasks._task_utils.file_utils import generate_submission_file
+from pathlib import Path
+import yaml
+import sys
+from typing import List, Dict, Optional, Union
+import re
+import cv2
+import numpy as np
+from loguru import logger as eval_logger
+
+TASK_TYPES = [
+    "TR",
+    "AR",
+    "VS",
+    "NQA",
+    "ER",
+    "PQA",
+    "SSC",
+    "AO",
+    "AC"
+]
+
+
+
+hf_home = os.getenv("HF_HOME", "./~/.cache/huggingface")
+base_cache_dir = os.path.expanduser(hf_home)
+
+with open(Path(__file__).parent / "mlvu.yaml", "r") as f:
+    raw_data = f.readlines()
+    safe_data = []
+    for i, line in enumerate(raw_data):
+        # remove function definition since yaml load cannot handle it
+        if "!function" not in line:
+            safe_data.append(line)
+cache_name = yaml.safe_load("".join(safe_data))["dataset_kwargs"]["cache_dir"]
+
+
+
+def mlvu_doc_to_visual(doc):
+
+    cache_dir = os.path.join(base_cache_dir, cache_name)
+    video_path = doc["video_name"]
+    video_path = os.path.join(cache_dir, video_path)
+    if os.path.exists(video_path):
+        video_path = video_path
+    else:
+        sys.exit(f"video path:{video_path} does not exist, please check")
+    return [video_path]
+
+
+def mlvu_doc_to_text(doc, model_specific_prompt_kwargs=None):
+    # option_prompt="Carefully watch this video and pay attention to every detail. Based on your observations, select the best option that accurately addresses the question."
+    option_prompt=""
+    question = doc["question"] + "\nOnly give the best option.\n" 
+    full_prompt=option_prompt+"\n"+question+"\n"+"Best option: ("
+    return full_prompt
+
+
+def extract_characters_regex(s):
+    s = s.strip()
+    if ")" in s:
+        index=s.index(")")
+        pred=s[index-1:index]
+        return pred
+    else:
+        return s
+
+def mlvu_process_results(doc, results):
+    """
+    Args:
+        doc: a instance of the eval dataset
+        results: [pred]
+    Returns:
+        a dictionary with key: metric name (in this case videomme score), value: metric value
+    """
+    pred = results[0]
+    # print("****************",pred)
+    pred_ans = extract_characters_regex(pred)
+
+    task_type = doc["task_type"]
+    data_dict = {"question_id": doc["question"], "task_type": task_type, "pred_answer": pred_ans, "answer": doc["answer"]}
+
+    return {f"mlvu_percetion_score": data_dict}
+
+
+def mlvu_aggregate_results(results):
+    """
+    Args:
+        results: a list of values returned by process_results
+    Returns:
+        A score
+    """
+    category2score = {}
+    for task_type in TASK_TYPES:
+        category2score[task_type] = {"correct": 0, "answered": 0}
+
+
+    for result in results:
+        task_type = result["task_type"]
+        category2score[task_type]["answered"] += 1
+        category2score[task_type]["correct"] += result["pred_answer"] == result["answer"]
+
+
+    for task_cate in TASK_TYPES:
+        total_correct = 0
+        total_answered = 0
+        for k, v in category2score.items():
+            if task_cate in k:
+                total_correct += v["correct"]
+                total_answered += v["answered"]
+        eval_logger.info(f"Evaluation on Task Categories: {task_cate}: {100 * total_correct / total_answered if total_answered > 0 else 0 : .1f}%")
+
+    total_correct = 0
+    total_answered = 0
+    for k, v in category2score.items():
+        total_correct += v["correct"]
+        total_answered += v["answered"]
+    eval_logger.info(f"Overall Performance: {100 * total_correct / total_answered if total_answered > 0 else 0 : .1f}%")
+
+    return 100 * total_correct / total_answered if total_answered > 0 else 0