Hugging Face's logo

NekoMikoReimu
/
top200mystery

Text Generation
Transformers
PyTorch
llama
Generated from Trainer
text-generation-inference
Inference Endpoints
Model card Files Community
NekoMikoReimu commited on
Commit
2858314
•
1 Parent(s): a9a11f3

Training in progress, epoch 1

Browse files
Files changed (4) hide show
  1. special_tokens_map.json +3 -3
  2. tokenizer.json +0 -27
  3. tokenizer_config.json +3 -27
  4. training_args.bin +1 -1
special_tokens_map.json CHANGED
@@ -1,13 +1,13 @@
1
  {
2
  "bos_token": {
3
- "content": "<s>",
4
  "lstrip": false,
5
  "normalized": false,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
- "content": "</s>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
@@ -21,7 +21,7 @@
21
  "single_word": false
22
  },
23
  "unk_token": {
24
- "content": "<unk>",
25
  "lstrip": false,
26
  "normalized": false,
27
  "rstrip": false,
 
1
  {
2
  "bos_token": {
3
+ "content": "<|endoftext|>",
4
  "lstrip": false,
5
  "normalized": false,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
+ "content": "<|endoftext|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
 
21
  "single_word": false
22
  },
23
  "unk_token": {
24
+ "content": "<|endoftext|>",
25
  "lstrip": false,
26
  "normalized": false,
27
  "rstrip": false,
tokenizer.json CHANGED
@@ -29,33 +29,6 @@
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
32
- },
33
- {
34
- "id": 65001,
35
- "content": "<s>",
36
- "single_word": false,
37
- "lstrip": false,
38
- "rstrip": false,
39
- "normalized": false,
40
- "special": true
41
- },
42
- {
43
- "id": 65002,
44
- "content": "</s>",
45
- "single_word": false,
46
- "lstrip": false,
47
- "rstrip": false,
48
- "normalized": false,
49
- "special": true
50
- },
51
- {
52
- "id": 65003,
53
- "content": "<unk>",
54
- "single_word": false,
55
- "lstrip": false,
56
- "rstrip": false,
57
- "normalized": false,
58
- "special": true
59
  }
60
  ],
61
  "normalizer": {
 
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  }
33
  ],
34
  "normalizer": {
tokenizer_config.json CHANGED
@@ -24,39 +24,15 @@
24
  "rstrip": false,
25
  "single_word": false,
26
  "special": true
27
- },
28
- "65001": {
29
- "content": "<s>",
30
- "lstrip": false,
31
- "normalized": false,
32
- "rstrip": false,
33
- "single_word": false,
34
- "special": true
35
- },
36
- "65002": {
37
- "content": "</s>",
38
- "lstrip": false,
39
- "normalized": false,
40
- "rstrip": false,
41
- "single_word": false,
42
- "special": true
43
- },
44
- "65003": {
45
- "content": "<unk>",
46
- "lstrip": false,
47
- "normalized": false,
48
- "rstrip": false,
49
- "single_word": false,
50
- "special": true
51
  }
52
  },
53
- "bos_token": "<s>",
54
  "clean_up_tokenization_spaces": true,
55
- "eos_token": "</s>",
56
  "model_max_length": 1000000000000000019884624838656,
57
  "pad_token": "[PAD]",
58
  "tokenizer_class": "GPTNeoXTokenizer",
59
  "trust_remote_code": false,
60
- "unk_token": "<unk>",
61
  "use_fast": true
62
  }
 
24
  "rstrip": false,
25
  "single_word": false,
26
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  }
28
  },
29
+ "bos_token": "<|endoftext|>",
30
  "clean_up_tokenization_spaces": true,
31
+ "eos_token": "<|endoftext|>",
32
  "model_max_length": 1000000000000000019884624838656,
33
  "pad_token": "[PAD]",
34
  "tokenizer_class": "GPTNeoXTokenizer",
35
  "trust_remote_code": false,
36
+ "unk_token": "<|endoftext|>",
37
  "use_fast": true
38
  }
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:40caa3ffe88e39fb8f17ca2f4b2952df2344fe3de9435a3b5cb8662a65ff745d
3
  size 6011
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86756e665c69be190c3de2e236f3cb1a2a4bdb410bba7277fd198efa1b9a6a27
3
  size 6011