adad

tokenizeConfig.py

@@ -21,6 +21,17 @@ class OBITokenizer(PreTrainedTokenizer):
         self.tokenizer.decoder = decoders.ByteLevel()
         super().__init__(**kwargs)
 
+        # Set the padding token
+        self.pad_token = "[PAD]"
+        # Set the special tokens
+        self.cls_token = "[CLS]"
+        self.sep_token = "[SEP]"
+        self.unk_token = "[UNK]"
+        self.mask_token = "[MASK]"
+        self.bos_token = "[CLS]"
+        self.eos_token = "[SEP]"
+        self.pad_token = "[PAD]"
+      
 
     def _pad(
             self,