-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathconfig.json
115 lines (115 loc) · 4.3 KB
/
config.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
{
"chat_templates": {
"llama-3": {
"init": "<|begin_of_text|>",
"roles as string": true,
"role start": "<|start_header_id|>",
"role end": "<|end_header_id|>\n\n",
"image token": "<image>\n",
"end text": "<|eot_id|>"
},
"gemma-2": {
"init": "<bos>",
"roles as string": true,
"role start": "<start_of_turn>",
"role end": "\n",
"image token": "<image>\n",
"end text": "<end_of_turn>\n",
"user role": "user",
"ai role": "model"
},
"hermes-2": {
"init": "",
"roles as string": true,
"role start": "<|im_start|>",
"role end": "\n",
"image token": "",
"end text": "<|im_end|>\n"
},
"phi-3": {
"init": "",
"roles as string": false,
"system role": "<|system|>\n",
"user role": "<|user|>\n",
"ai role": "<|assistant|>\n",
"image token": "<|image_1|>\n",
"end text": "<|end|>\n"
}
},
"models": {
"gemma-2-9b": {
"name": "gemma-2-9b",
"path": "/home/tobi/ai/models/text_to_text/gemma-2-9b-it",
"template": "gemma-2",
"backend": "transformers",
"image-capable": false
},
"Meta-Llama-3-70B-Instruct-2.25bpw-h6-exl2": {
"name": "Meta-Llama-3-70B-Instruct-2.25bpw-h6-exl2",
"path": "/home/tobi/ai/models/text_to_text/exllamav2/Meta-Llama-3-70B-Instruct-2.25bpw-h6-exl2",
"template": "llama-3",
"backend": "exllamav2",
"image-capable": false
},
"Meta-Llama-3-8B-Instruct-6.0bpw-h6-exl2": {
"name": "Meta-Llama-3-8B-Instruct-6.0bpw-h6-exl2",
"path": "/home/tobi/ai/models/text_to_text/exllamav2/Meta-Llama-3-8B-Instruct-6.0bpw-h6-exl2",
"template": "llama-3",
"backend": "exllamav2",
"image-capable": false
},
"llama3-llava-next-8b": {
"name": "llama3-llava-next-8b",
"path": "/home/tobi/ai/models/multimodal/transformers/llama3-llava-next-8b",
"template": "llama-3",
"backend": "transformers",
"image-capable": true
},
"Meta-Llama-3-70B-Instruct-IQ2_S": {
"name": "Meta-Llama-3-70B-Instruct-IQ2_S",
"path": "/home/tobi/ai/models/text_to_text/llama-cpp/Meta-Llama-3-70B-Instruct-IQ2_S.gguf",
"template": "llama-3",
"backend": "llama-cpp",
"image-capable": false
},
"Meta-Llama-3-70B-Instruct-IQ1_M": {
"name": "Meta-Llama-3-70B-Instruct-IQ1_M",
"path": "/home/tobi/ai/models/text_to_text/llama-cpp/Meta-Llama-3-70B-Instruct-IQ1_M.gguf",
"template": "llama-3",
"backend": "llama-cpp",
"image-capable": false
},
"Hermes-2-Theta-Llama-3-8B": {
"name": "Hermes-2-Theta-Llama-3-8B",
"path": "/home/tobi/ai/models/text_to_text/transformers/Hermes-2-Theta-Llama-3-8B",
"template": "hermes-2",
"backend": "transformers",
"image-capable": false
},
"phi-3-mini-128k-instruct": {
"name": "phi-3-mini-128k-instruct",
"path": "/home/tobi/ai/models/text_to_text/transformers/Phi-3-mini-128k-instruct",
"template": "phi-3",
"backend": "transformers",
"image-capable": false
},
"phi-3-medium-128k-instruct": {
"name": "phi-3-medium-128k-instruct",
"path": "/home/tobi/ai/models/text_to_text/transformers/Phi-3-medium-128k-instruct",
"template": "phi-3",
"backend": "transformers",
"image-capable": false
},
"phi-3-vision-128k-instruct": {
"name": "phi-3-vision-128k-instruct",
"path": "/home/tobi/ai/models/multimodal/transformers/Phi-3-vision-128k-instruct",
"template": "phi-3",
"backend": "transformers",
"image-capable": true
}
},
"max_new_tokens": 1024,
"torch_device": "cuda",
"torch_device_map": "cuda",
"torch_cuda_garbage_collection_threshold": 0.01
}