tuanle commited on
Commit
640f858
1 Parent(s): 9a31a59

add tokenizer

Browse files
Files changed (2) hide show
  1. added_tokens.json +1 -1
  2. tokenizer.json +2 -2
added_tokens.json CHANGED
@@ -1 +1 @@
1
- {"<|endoftext|>": 50257, "\n": 50258}
 
1
+ {"\n": 50258, "<|endoftext|>": 50257}
tokenizer.json CHANGED
@@ -59,12 +59,12 @@
59
  },
60
  {
61
  "id": 50258,
62
- "special": false,
63
  "content": "\n",
64
  "single_word": false,
65
  "lstrip": false,
66
  "rstrip": false,
67
- "normalized": true
68
  }
69
  ],
70
  "normalizer": null,
 
59
  },
60
  {
61
  "id": 50258,
62
+ "special": true,
63
  "content": "\n",
64
  "single_word": false,
65
  "lstrip": false,
66
  "rstrip": false,
67
+ "normalized": false
68
  }
69
  ],
70
  "normalizer": null,