forked from zhao-ht/GIMLET
-
Notifications
You must be signed in to change notification settings - Fork 0
/
downstream_test.sh
115 lines (88 loc) · 6.23 KB
/
downstream_test.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
#!/bin/bash
device=$1
model=$2
model_ckpt_name=$3
few_shot_number=$4
prompt_augmentation=$5
ckpt_dir="ckpts/"
if [ ! -d cache ];then
mkdir cache
fi
out_file_prefix="cache/testing_"
outfile=$out_file_prefix${model_ckpt_name//'/'/'_'}".csv"
echo "Output results to: "$outfile
export dataset_list=(bace hiv muv tox21 toxcast bbbp cyp450 pcba esol lipo freesolv)
if [ $model = 'gimlet' ]
then
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device python downstream_test.py --zero_shot --transformer_backbone gimlet --model_name_or_path $ckpt_dir$model_ckpt_name --tokenizer_name t5-small --dataset "$dataset" --runseed 5 --batch_size 40 --grad_accum_step 1 --transform_in_collator --only_test --output_result_to_file $outfile --not_retest_tasks_in_result_file
done
elif [ $model = 'gimlet_fewshot' ]
then
out_file_prefix=$out_file_prefix"few_shot_"$few_shot_number"_"
outfile=$out_file_prefix${model_ckpt_name//'/'/'_'}".csv"
echo $outfile
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device python downstream_test.py --few_shot $few_shot_number --few_shot_fashion linear --transformer_backbone gimlet --model_name_or_path $ckpt_dir$model_ckpt_name --tokenizer_name t5-small --dataset "$dataset" --runseed 5 --batch_size 128 --grad_accum_step 8 --lr 1e-2 --epochs 60 --test_interval 20 --transform_in_collator --no_eval_train --output_result_to_file $outfile --not_retest_tasks_in_result_file
done
elif [ $model = 'gimlet_aug' ]
then
out_file_prefix=$out_file_prefix"augment_"$prompt_augmentation"_"
outfile=$out_file_prefix${model_ckpt_name//'/'/'_'}".csv"
echo $outfile
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device python downstream_test.py --zero_shot --transformer_backbone gimlet --model_name_or_path $ckpt_dir$model_ckpt_name --tokenizer_name t5-small --dataset "$dataset" --prompt_file augmented_selected_prompt_downstream_task.json --runseed 5 --batch_size 40 --grad_accum_step 1 --transform_in_collator --only_test --output_result_to_file $outfile --not_retest_tasks_in_result_file --prompt_augmentation $prompt_augmentation
done
elif [ $model = 'kvplm' ]
then
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device python downstream_test.py --zero_shot --transformer_backbone kvplm --model_name_or_path $ckpt_dir$model_ckpt_name --dataset "$dataset" --prompt_file prompt_downstream_task.json --runseed 5 --batch_size 40 --grad_accum_step 1 --only_test --output_result_to_file $outfile --not_retest_tasks_in_result_file
done
elif [ $model = 'kvplm_fewshot' ]
then
out_file_prefix=$out_file_prefix"few_shot_"$few_shot_number"_"
outfile=$out_file_prefix${model_ckpt_name//'/'/'_'}".csv"
echo $outfile
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device python downstream_test.py --few_shot $few_shot_number --few_shot_fashion linear --transformer_backbone kvplm --model_name_or_path $ckpt_dir$model_ckpt_name --dataset "$dataset" --prompt_file prompt_downstream_task.json --runseed 5 --batch_size 128 --grad_accum_step 8 --lr 1e-2 --epochs 60 --test_interval 20 --no_eval_train --output_result_to_file $outfile --not_retest_tasks_in_result_file
done
elif [ $model = 'kvplm_aug' ]
then
out_file_prefix=$out_file_prefix"augment_"$prompt_augmentation"_"
outfile=$out_file_prefix${model_ckpt_name//'/'/'_'}".csv"
echo $outfile
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device python downstream_test.py --zero_shot --transformer_backbone kvplm --model_name_or_path $ckpt_dir$model_ckpt_name --dataset "$dataset" --prompt_file augmented_prompt_downstream_task.json --runseed 5 --batch_size 40 --grad_accum_step 1 --only_test --output_result_to_file $outfile --not_retest_tasks_in_result_file --prompt_augmentation $prompt_augmentation
done
elif [ $model = 'momu' ]
then
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device python downstream_test.py --zero_shot --transformer_backbone momu --model_name_or_path $ckpt_dir$model_ckpt_name --dataset "$dataset" --prompt_file prompt_downstream_task.json --runseed 5 --batch_size 40 --grad_accum_step 1 --only_test --output_result_to_file $outfile --not_retest_tasks_in_result_file
done
elif [ $model = 'momu_fewshot' ]
then
out_file_prefix=$out_file_prefix"few_shot_"$few_shot_number"_"
outfile=$out_file_prefix${model_ckpt_name//'/'/'_'}".csv"
echo $outfile
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device python downstream_test.py --few_shot $few_shot_number --few_shot_fashion linear --transformer_backbone momu --model_name_or_path $ckpt_dir$model_ckpt_name --dataset "$dataset" --prompt_file prompt_downstream_task.json --runseed 5 --batch_size 128 --grad_accum_step 8 --lr 1e-2 --epochs 60 --test_interval 20 --no_eval_train --output_result_to_file $outfile --not_retest_tasks_in_result_file
done
elif [ $model = 'momu_aug' ]
then
out_file_prefix=$out_file_prefix"augment_"$prompt_augmentation"_"
outfile=$out_file_prefix${model_ckpt_name//'/'/'_'}".csv"
echo $outfile
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device python downstream_test.py --zero_shot --transformer_backbone momu --model_name_or_path $ckpt_dir$model_ckpt_name --dataset "$dataset" --prompt_file augmented_prompt_downstream_task.json --runseed 5 --batch_size 40 --grad_accum_step 1 --only_test --output_result_to_file $outfile --not_retest_tasks_in_result_file --prompt_augmentation $prompt_augmentation
done
elif [ $model = 'galactica' ]
then
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device python downstream_test.py --zero_shot --transformer_backbone galactica --model_name_or_path $model_ckpt_name --dataset "$dataset" --prompt_file prompt_downstream_task.json --runseed 5 --batch_size 1 --grad_accum_step 1 --only_test --output_result_to_file $outfile --not_retest_tasks_in_result_file
done
elif [ $model = 'gpt3' ]
then
for dataset in "${dataset_list[@]}"; do
CUDA_VISIBLE_DEVICES=$device OPENAI_API_KEY=$openai_key python downstream_test.py --dataset "$dataset" --runseed 5 --eval_train --batch_size 1 --grad_accum_step 1 --lr 1e-5 --epochs 50 --transformer_backbone gpt3 --model_name_or_path $model_ckpt_name --zero_shot --output_result_to_file $outfile --only_test
done
fi