File size: 412 Bytes
43404e0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
{
	"bos_token": {
		"content": "<|begin_of_text|>",
		"lstrip": false,
		"normalized": false,
		"rstrip": false,
		"single_word": false
	},
	"eos_token": {
		"content": "<|end_of_text|>",
		"lstrip": false,
		"normalized": false,
		"rstrip": false,
		"single_word": false
	},
	"pad_token": {
		"content": "<|end_of_text|>",
		"lstrip": false,
		"normalized": false,
		"rstrip": false,
		"single_word": false
	}
}