adsa

tokenizeConfig.py

@@ -1,6 +1,11 @@
 from transformers import PreTrainedTokenizer
 from tokenizers import Tokenizer, models, pre_tokenizers, trainers, decoders
 import json
+from typing import List, Optional, Union, Dict
+from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
+from transformers import PreTrainedTokenizer
+from transformers.utils import logging, PaddingStrategy
+
 
 class OBITokenizer(PreTrainedTokenizer):
     def __init__(self, auto_map={"AutoTokenizer": ["tokenizeConfig.OBITokenizer"]},tokenizer_class= "OBITokenizer",**kwargs):
@@ -16,6 +21,20 @@ class OBITokenizer(PreTrainedTokenizer):
         self.tokenizer.decoder = decoders.ByteLevel()
         super().__init__(**kwargs)
 
+
+    def _pad(
+            self,
+            encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
+            max_length: Optional[int] = None,
+            padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
+            pad_to_multiple_of: Optional[int] = None,
+            return_attention_mask: Optional[bool] = None,
+    ) -> dict:
+        # Modify the _pad method as needed for OBITokenizer
+        # You can inherit the implementation from ChatGLMTokenizer and customize it further
+        return super()._pad(encoded_inputs, max_length, padding_strategy, pad_to_multiple_of, return_attention_mask)
+        
+
     def train(self, files,save_path):
         # Training: Fit the tokenizer on your text data
         trainer = trainers.BpeTrainer(special_tokens=["[PAD]", "[CLS]", "[SEP]", "[MASK]", "[UNK]"])
@@ -32,7 +51,7 @@ class OBITokenizer(PreTrainedTokenizer):
             "vocab_size": self.tokenizer.get_vocab_size(),
             "tokenizer_class": "OBITokenizer",
             "auto_map": {
-                "AutoTokenizer": ["tokenizeConfig.OBITokenizer"]
+                "AutoTokenizer": ["tokenizeConfig.OBITokenizer","null"]
             },
             "bos_token": "[CLS]",
             "eos_token": "[SEP]",