-
Notifications
You must be signed in to change notification settings - Fork 1
/
train_4folds_and_final.sh
132 lines (109 loc) · 10.5 KB
/
train_4folds_and_final.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
DATA_DIR=$1
echo "**********Train Bootstrap UnetEffnetV2L**********"
CUDA_VISIBLE_DEVICES=0 PYTHONPATH=. python -u train_fold.py --gpu 0 --workers 4 --fold 0 \
--config configs/bootstrap_v2l.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/0 &
CUDA_VISIBLE_DEVICES=1 PYTHONPATH=. python -u train_fold.py --gpu 1 --workers 4 --fold 1 \
--config configs/bootstrap_v2l.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/1 &
CUDA_VISIBLE_DEVICES=2 PYTHONPATH=. python -u train_fold.py --gpu 2 --workers 4 --fold 2 \
--config configs/bootstrap_v2l.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/2 &
CUDA_VISIBLE_DEVICES=3 PYTHONPATH=. python -u train_fold.py --gpu 3 --workers 4 --fold 3 \
--config configs/bootstrap_v2l.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/3 &
wait
echo "**********Train Bootstrap UnetEffnetV2L finished**********"
echo "**********Train Bootstrap UnetEffnetL2**********"
CUDA_VISIBLE_DEVICES=0 PYTHONPATH=. python -u train_fold.py --gpu 0 --workers 4 --fold 0 \
--config configs/bootstrap_l2.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/0 &
CUDA_VISIBLE_DEVICES=1 PYTHONPATH=. python -u train_fold.py --gpu 1 --workers 4 --fold 1 \
--config configs/bootstrap_l2.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/1 &
CUDA_VISIBLE_DEVICES=2 PYTHONPATH=. python -u train_fold.py --gpu 2 --workers 4 --fold 2 \
--config configs/bootstrap_l2.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/2 &
CUDA_VISIBLE_DEVICES=3 PYTHONPATH=. python -u train_fold.py --gpu 3 --workers 4 --fold 3 \
--config configs/bootstrap_l2.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/3 &
wait
echo "**********Train Bootstrap UnetEffnetL2 finished**********"
echo "**********Train Bootstrap UnetMaxvit**********"
CUDA_VISIBLE_DEVICES=0 PYTHONPATH=. python -u train_fold.py --gpu 0 --workers 4 --fold 0 \
--config configs/bootstrap_maxvit.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/0 &
CUDA_VISIBLE_DEVICES=1 PYTHONPATH=. python -u train_fold.py --gpu 1 --workers 4 --fold 1 \
--config configs/bootstrap_maxvit.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/1 &
CUDA_VISIBLE_DEVICES=2 PYTHONPATH=. python -u train_fold.py --gpu 2 --workers 4 --fold 2 \
--config configs/bootstrap_maxvit.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/2 &
CUDA_VISIBLE_DEVICES=3 PYTHONPATH=. python -u train_fold.py --gpu 3 --workers 4 --fold 3 \
--config configs/bootstrap_maxvit.json --test_every 1 --data-dir $DATA --prefix bootstrap_ --fp16 > logs/3 &
wait
echo "**********Train Bootstrap UnetMaxvit finished**********"
echo "**********Predicting pseudo masks**********"
CUDA_VISIBLE_DEVICES=0 PYTHONPATH=. python predict_pseudo_simple.py --gpu 0 --config bootstrap_l2,bootstrap_maxvit,bootstrap_v2l --weights-path weights --data-dir $DATA_DIR --out-dir $DATA_DIR/seg_preds \
--checkpoint bootstrap_TimmUnetPure_tf_efficientnet_l2_ns_0_dice,bootstrap_TimmUnetPure_maxvit_base_tf_512.in21k_ft_in1k_0_dice,bootstrap_TimmUnetPure_tf_efficientnetv2_l_in21k_0_dice --fold 0 > logs/0 &
CUDA_VISIBLE_DEVICES=1 PYTHONPATH=. python predict_pseudo_simple.py --gpu 1 --config bootstrap_l2,bootstrap_maxvit,bootstrap_v2l --weights-path weights --data-dir $DATA_DIR --out-dir $DATA_DIR/seg_preds \
--checkpoint bootstrap_TimmUnetPure_tf_efficientnet_l2_ns_1_dice,bootstrap_TimmUnetPure_maxvit_base_tf_512.in21k_ft_in1k_1_dice,bootstrap_TimmUnetPure_tf_efficientnetv2_l_in21k_1_dice --fold 1 > logs/1 &
CUDA_VISIBLE_DEVICES=2 PYTHONPATH=. python predict_pseudo_simple.py --gpu 2 --config bootstrap_l2,bootstrap_maxvit,bootstrap_v2l --weights-path weights --data-dir $DATA_DIR --out-dir $DATA_DIR/seg_preds \
--checkpoint bootstrap_TimmUnetPure_tf_efficientnet_l2_ns_2_dice,bootstrap_TimmUnetPure_maxvit_base_tf_512.in21k_ft_in1k_2_dice,bootstrap_TimmUnetPure_tf_efficientnetv2_l_in21k_2_dice --fold 2 > logs/2 &
CUDA_VISIBLE_DEVICES=3 PYTHONPATH=. python predict_pseudo_simple.py --gpu 3 --config bootstrap_l2,bootstrap_maxvit,bootstrap_v2l --weights-path weights --data-dir $DATA_DIR --out-dir $DATA_DIR/seg_preds \
--checkpoint bootstrap_TimmUnetPure_tf_efficientnet_l2_ns_3_dice,bootstrap_TimmUnetPure_maxvit_base_tf_512.in21k_ft_in1k_3_dice,bootstrap_TimmUnetPure_tf_efficientnetv2_l_in21k_3_dice --fold 3 > logs/3 &
wait
echo "**********Finished predicting**********"
echo "**********Train Round UnetEffnetV2L**********"
CUDA_VISIBLE_DEVICES=0 PYTHONPATH=. python -u train_fold.py --gpu 0 --workers 4 --fold 0 \
--config configs/round_v2l.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/0 &
CUDA_VISIBLE_DEVICES=1 PYTHONPATH=. python -u train_fold.py --gpu 1 --workers 4 --fold 1 \
--config configs/round_v2l.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/1 &
CUDA_VISIBLE_DEVICES=2 PYTHONPATH=. python -u train_fold.py --gpu 2 --workers 4 --fold 2 \
--config configs/round_v2l.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/2 &
CUDA_VISIBLE_DEVICES=3 PYTHONPATH=. python -u train_fold.py --gpu 3 --workers 4 --fold 3 \
--config configs/round_v2l.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/3 &
wait
echo "**********Train Round UnetEffnetV2L finished**********"
echo "**********Train Round UnetEffnetL2**********"
CUDA_VISIBLE_DEVICES=0 PYTHONPATH=. python -u train_fold.py --gpu 0 --workers 4 --fold 0 \
--config configs/round_l2.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/0 &
CUDA_VISIBLE_DEVICES=1 PYTHONPATH=. python -u train_fold.py --gpu 1 --workers 4 --fold 1 \
--config configs/round_l2.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/1 &
CUDA_VISIBLE_DEVICES=2 PYTHONPATH=. python -u train_fold.py --gpu 2 --workers 4 --fold 2 \
--config configs/round_l2.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/2 &
CUDA_VISIBLE_DEVICES=3 PYTHONPATH=. python -u train_fold.py --gpu 3 --workers 4 --fold 3 \
--config configs/round_l2.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/3 &
wait
echo "**********Train Round UnetEffnetL2 finished**********"
echo "**********Train Round UnetMaxvit**********"
CUDA_VISIBLE_DEVICES=0 PYTHONPATH=. python -u train_fold.py --gpu 0 --workers 4 --fold 0 \
--config configs/round_maxvit.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/0 &
CUDA_VISIBLE_DEVICES=1 PYTHONPATH=. python -u train_fold.py --gpu 1 --workers 4 --fold 1 \
--config configs/round_maxvit.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/1 &
CUDA_VISIBLE_DEVICES=2 PYTHONPATH=. python -u train_fold.py --gpu 2 --workers 4 --fold 2 \
--config configs/round_maxvit.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/2 &
CUDA_VISIBLE_DEVICES=3 PYTHONPATH=. python -u train_fold.py --gpu 3 --workers 4 --fold 3 \
--config configs/round_maxvit.json --test_every 1 --data-dir $DATA --prefix round_ --fp16 > logs/3 &
wait
echo "**********Train Round UnetMaxvit finished**********"
echo "**********Predicting pseudo masks**********"
CUDA_VISIBLE_DEVICES=0 PYTHONPATH=. python predict_pseudo_simple.py --gpu 0 --config round_l2,round_maxvit,round_v2l --weights-path weights --data-dir $DATA_DIR --out-dir $DATA_DIR/seg_preds \
--checkpoint round_TimmUnetPure_tf_efficientnet_l2_ns_0_dice,round_TimmUnetPure_maxvit_base_tf_512.in21k_ft_in1k_0_dice,round_TimmUnetPure_tf_efficientnetv2_l_in21k_0_dice --fold 0 > logs/0 &
CUDA_VISIBLE_DEVICES=1 PYTHONPATH=. python predict_pseudo_simple.py --gpu 1 --config round_l2,round_maxvit,round_v2l --weights-path weights --data-dir $DATA_DIR --out-dir $DATA_DIR/seg_preds \
--checkpoint round_TimmUnetPure_tf_efficientnet_l2_ns_1_dice,round_TimmUnetPure_maxvit_base_tf_512.in21k_ft_in1k_1_dice,round_TimmUnetPure_tf_efficientnetv2_l_in21k_1_dice --fold 1 > logs/1 &
CUDA_VISIBLE_DEVICES=2 PYTHONPATH=. python predict_pseudo_simple.py --gpu 2 --config round_l2,round_maxvit,round_v2l --weights-path weights --data-dir $DATA_DIR --out-dir $DATA_DIR/seg_preds \
--checkpoint round_TimmUnetPure_tf_efficientnet_l2_ns_2_dice,round_TimmUnetPure_maxvit_base_tf_512.in21k_ft_in1k_2_dice,round_TimmUnetPure_tf_efficientnetv2_l_in21k_2_dice --fold 2 > logs/2 &
CUDA_VISIBLE_DEVICES=3 PYTHONPATH=. python predict_pseudo_simple.py --gpu 3 --config round_l2,round_maxvit,round_v2l --weights-path weights --data-dir $DATA_DIR --out-dir $DATA_DIR/seg_preds \
--checkpoint round_TimmUnetPure_tf_efficientnet_l2_ns_3_dice,round_TimmUnetPure_maxvit_base_tf_512.in21k_ft_in1k_3_dice,round_TimmUnetPure_tf_efficientnetv2_l_in21k_3_dice --fold 3 > logs/3 &
wait
echo "**********Finished predicting**********"
CUDA_VISIBLE_DEVICES=0,1,2,3 PYTHONPATH=. python -u -m torch.distributed.launch --nproc_per_node=4 \
--master_port 9972 train_all.py --world-size 4 --distributed --workers 16 --fold 0 \
--config configs/all_l2.json --test_every 1 --fp16 --prefix final_ > logs/l2 &
wait
CUDA_VISIBLE_DEVICES=0,1,2,3 PYTHONPATH=. python -u -m torch.distributed.launch --nproc_per_node=4 \
--master_port 9972 train_all.py --world-size 4 --distributed --workers 16 --fold 0 \
--config configs/all_v2l.json --test_every 1 --fp16 --prefix final_ > logs/v2l &
wait
CUDA_VISIBLE_DEVICES=0,1,2,3 PYTHONPATH=. python -u -m torch.distributed.launch --nproc_per_node=4 \
--master_port 9972 train_all.py --world-size 4 --distributed --workers 16 --fold 0 \
--config configs/all_v2xl.json --test_every 1 --fp16 --prefix final_ > logs/v2xl &
wait
CUDA_VISIBLE_DEVICES=0,1,2,3 PYTHONPATH=. python -u -m torch.distributed.launch --nproc_per_node=4 \
--master_port 9972 train_all.py --world-size 4 --distributed --workers 16 --fold 0 \
--config configs/all_maxvit.json --test_every 1 --fp16 --prefix final_ > logs/maxvit &
wait
python unet_swa.py --path weights --exp_name final_TimmUnetPure_tf_efficientnetv2_xl_in21k_0 --num_best 5 --min_epoch 25 --maximize --metric_name dice
python unet_swa.py --path weights --exp_name final_TimmUnetPure_tf_efficientnetv2_l_in21k_0 --num_best 5 --min_epoch 25 --maximize --metric_name dice
python unet_swa.py --path weights --exp_name final_TimmUnetPure_tf_efficientnet_l2_ns_0 --num_best 5 --min_epoch 25 --maximize --metric_name dice
python unet_swa.py --path weights --exp_name final_TimmUnetPure_maxvit_base_tf_512.in21k_ft_in1k_0 --num_best 5 --min_epoch 25 --maximize --metric_name dice