Pragaash commited on
Commit
a9a132f
1 Parent(s): 72e0b3e

Fix auto tokenizer import reference format in auto map as list for slow and fast.

Browse files
Files changed (2) hide show
  1. config.json +4 -1
  2. tokenizer_config.json +4 -1
config.json CHANGED
@@ -12,7 +12,10 @@
12
  "auto_map": {
13
  "AutoConfig": "configuration_hyena.StripedHyenaConfig",
14
  "AutoModelForCausalLM": "modeling_hyena.StripedHyenaModelForCausalLM",
15
- "AutoTokenizer": "tokenizer.ByteTokenizer"
 
 
 
16
  },
17
  "column_split": false,
18
  "column_split_hyena": true,
 
12
  "auto_map": {
13
  "AutoConfig": "configuration_hyena.StripedHyenaConfig",
14
  "AutoModelForCausalLM": "modeling_hyena.StripedHyenaModelForCausalLM",
15
+ "AutoTokenizer": [
16
+ "tokenizer.ByteTokenizer",
17
+ null
18
+ ]
19
  },
20
  "column_split": false,
21
  "column_split_hyena": true,
tokenizer_config.json CHANGED
@@ -1,7 +1,10 @@
1
  {
2
  "added_tokens_decoder": {},
3
  "auto_map": {
4
- "AutoTokenizer": "tokenizer.ByteTokenizer"
 
 
 
5
  },
6
  "byte_level": true,
7
  "clean_up_tokenization_spaces": true,
 
1
  {
2
  "added_tokens_decoder": {},
3
  "auto_map": {
4
+ "AutoTokenizer": [
5
+ "tokenizer.ByteTokenizer",
6
+ null
7
+ ]
8
  },
9
  "byte_level": true,
10
  "clean_up_tokenization_spaces": true,