- tokenizer_config.json +2 -2
tokenizer_config.json
CHANGED
|
@@ -3,7 +3,7 @@
|
|
| 3 |
"add_eos_token": false,
|
| 4 |
"auto_map": {
|
| 5 |
"AutoTokenizer": [
|
| 6 |
-
"tokenizeConfig.
|
| 7 |
null
|
| 8 |
]
|
| 9 |
},
|
|
@@ -34,7 +34,7 @@
|
|
| 34 |
"single_word": true
|
| 35 |
},
|
| 36 |
"sp_model_kwargs": {},
|
| 37 |
-
"tokenizer_class": "
|
| 38 |
"unk_token": {
|
| 39 |
"__type": "AddedToken",
|
| 40 |
"content": "<unk>",
|
|
|
|
| 3 |
"add_eos_token": false,
|
| 4 |
"auto_map": {
|
| 5 |
"AutoTokenizer": [
|
| 6 |
+
"tokenizeConfig.OBTokenzier",
|
| 7 |
null
|
| 8 |
]
|
| 9 |
},
|
|
|
|
| 34 |
"single_word": true
|
| 35 |
},
|
| 36 |
"sp_model_kwargs": {},
|
| 37 |
+
"tokenizer_class": "OBTokenzier",
|
| 38 |
"unk_token": {
|
| 39 |
"__type": "AddedToken",
|
| 40 |
"content": "<unk>",
|