Fix tokenizer files
Browse files- tokenizer.json +1 -7
- tokenizer_config.json +0 -0
tokenizer.json
CHANGED
|
@@ -326,12 +326,6 @@
|
|
| 326 |
"id": "A",
|
| 327 |
"type_id": 0
|
| 328 |
}
|
| 329 |
-
},
|
| 330 |
-
{
|
| 331 |
-
"SpecialToken": {
|
| 332 |
-
"id": "<|end_text|>",
|
| 333 |
-
"type_id": 0
|
| 334 |
-
}
|
| 335 |
}
|
| 336 |
],
|
| 337 |
"pair": [
|
|
@@ -128597,4 +128591,4 @@
|
|
| 128597 |
"âĭ ¿"
|
| 128598 |
]
|
| 128599 |
}
|
| 128600 |
-
}
|
|
|
|
| 326 |
"id": "A",
|
| 327 |
"type_id": 0
|
| 328 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 329 |
}
|
| 330 |
],
|
| 331 |
"pair": [
|
|
|
|
| 128591 |
"âĭ ¿"
|
| 128592 |
]
|
| 128593 |
}
|
| 128594 |
+
}
|
tokenizer_config.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|