adjldh

tokenizeConfig.py

@@ -53,6 +53,11 @@ class OBITokenizer(PreTrainedTokenizer):
         # Load the vocabulary file
         self.tokenizer.get_vocab().add_special_tokens([self.cls_token, self.sep_token, self.unk_token, self.mask_token])
 
+    def add_special_tokens(self, special_tokens_dict):
+        # Implement the addition of special tokens to the vocabulary here
+        # Example: self.tokenizer.get_vocab().add_special_tokens(special_tokens_dict)
+        return self.tokenizer.get_vocab().add_special_tokens(special_tokens_dict)
+
     def _tokenize(self, text):
         # Implement your custom tokenization logic here
         # In this example, we split the text into tokens using whitespace