-
Notifications
You must be signed in to change notification settings - Fork 8
/
eval_FB13_ft.py
98 lines (78 loc) · 3.23 KB
/
eval_FB13_ft.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
labels = []
with open("data/FB13/test.tsv", "r") as f:
lines = f.readlines()
for line in lines:
tmp = line.strip().split("\t")
labels.append(tmp[3])
correct_count = 0
with open("data/FB13/pred_instructions_llama.csv", "r") as f:
lines = f.readlines()
print(len(lines))
for (i, line) in enumerate(lines[1:]):
line = line.strip()
start_idx = line.find("\"")
res = line[start_idx + 1: -1]
label = labels[i]
#print(res, label)
if res.find("Yes") != -1 and label == "1":
correct_count += 1
elif res.find("No") != -1 and label == "-1":
correct_count += 1
print("LLaMA-7B acc: ", correct_count, 1.0 * correct_count/len(labels))
correct_count = 0
with open("data/FB13/pred_instructions_llama13b.csv", "r") as f:
lines = f.readlines()
print(len(lines))
for (i, line) in enumerate(lines[1:]):
line = line.strip()
start_idx = line.find("\"")
res = line[start_idx + 1: -1]
label = labels[i]
#print(res, label)
if res.find("Yes") != -1 and label == "1":
correct_count += 1
elif res.find("No") != -1 and label == "-1":
correct_count += 1
print("LLaMA-13B acc: ", correct_count, 1.0 * correct_count/len(labels))
correct_count = 0
with open("data/FB13/pred_instructions_llama_raw.csv", "r", encoding="utf-8") as f:
lines = f.readlines()
for (i, line) in enumerate(lines[1:]):
line = line.strip()
start_idx = line.find("?")
res = line[start_idx + 1:]
label = labels[i]
if (res.find("Yes,") != -1 or res.find(" yes") != -1) and label == "1":
correct_count += 1
#print(line, res, label)
elif (res.find("No") != -1 or res.find("not") != -1 or res.find("n't") != -1 or res.find("no") != -1) and label == "-1":
correct_count += 1
print("LLaMA-7B raw acc: ", correct_count, 1.0 * correct_count/len(labels))
correct_count = 0
with open("data/FB13/pred_instructions_llama_raw13B.csv", "r", encoding="utf-8") as f:
lines = f.readlines()
for (i, line) in enumerate(lines[1:]):
line = line.strip()
start_idx = line.find("?")
res = line[start_idx + 1:]
label = labels[i]
if (res.find("Yes,") != -1 or res.find(" yes") != -1) and label == "1":
correct_count += 1
#print(line, res, label)
elif (res.find("No") != -1 or res.find("not") != -1 or res.find("n't") != -1 or res.find("no") != -1) and label == "-1":
correct_count += 1
print("LLaMA-13B raw acc: ", correct_count, 1.0 * correct_count/len(labels))
correct_count = 0
with open("data/FB13/generated_predictions.txt", "r") as f:
lines = f.readlines()
for (i, line) in enumerate(lines[1:]):
line = line.strip()
start_idx = line.find("\"predict\": \"")
res = line[start_idx + len("\"predict\": \""): -2]
label = labels[i]
#print(res, label)
if res.find("Yes") != -1 and label == "1":
correct_count += 1
elif res.find("No") != -1 and label == "-1":
correct_count += 1
print("GLM acc: ", correct_count, 1.0 * correct_count/len(labels))