ad

ads

tokenizeConfig.py

@@ -3,7 +3,10 @@ from tokenizers import Tokenizer, models, pre_tokenizers, trainers, decoders
 import json
 
 class OBITokenizer(PreTrainedTokenizer):
-    def __init__(self):
+    def __init__(self, auto_map={"AutoTokenizer": ["tokenizeConfig.OBITokenizer"]},tokenizer_class= "OBITokenizer",**kwargs):
+         # Initialize your tokenizer with the auto_map parameter if needed
+        self.auto_map=auto_map
+        self.tokenizer_class=tokenizer_class
         # Initialize a BPE model for tokenization
         bpe_model = models.BPE()
         # Initialize the tokenizer
@@ -11,6 +14,7 @@ class OBITokenizer(PreTrainedTokenizer):
         # Add pre-tokenization and decoding steps if needed
         self.tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
         self.tokenizer.decoder = decoders.ByteLevel()
+        super().__init__(**kwargs)
 
     def train(self, files,save_path):
         # Training: Fit the tokenizer on your text data