sds
Browse files- tokenizeConfig.py +1 -1
tokenizeConfig.py
CHANGED
|
@@ -20,7 +20,7 @@ PRETRAINED_VOCAB_FILES_MAP = {
|
|
| 20 |
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
|
| 21 |
|
| 22 |
|
| 23 |
-
class
|
| 24 |
"""
|
| 25 |
Construct a Baichuan tokenizer. Based on byte-level Byte-Pair-Encoding.
|
| 26 |
Args:
|
|
|
|
| 20 |
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
|
| 21 |
|
| 22 |
|
| 23 |
+
class OBTokenzier(PreTrainedTokenizer):
|
| 24 |
"""
|
| 25 |
Construct a Baichuan tokenizer. Based on byte-level Byte-Pair-Encoding.
|
| 26 |
Args:
|