dis

tokenizer_config.json

@@ -1,36 +1,19 @@
 {
-  "auto_map": {
-    "AutoTokenizer": ["tokenizeConfig.OBITokenizer", null]
-  },
-  "add_bos_token": false,
-  "add_eos_token": false,
-  "use_fast": false,
-  "clean_up_tokenization_spaces": false,
-  "eos_token": {
-    "__type": "AddedToken",
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": true
-  },
-  "model_max_length": 4096,
-  "sp_model_kwargs": {},
-  "tokenizer_class": "OBITokenizer",
-  "pad_token": {
-    "__type": "AddedToken",
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": true
-  },
-  "unk_token": {
-    "__type": "AddedToken",
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": true
-  }
+    "auto_map": {
+      "AutoTokenizer": ["tokenizeConfig.OBITokenizer", null]
+    },
+      "add_prefix_space": false,
+      "additional_special_tokens": [
+        "</s>",
+        "<s>",
+        "<unk>"
+      ],
+      "bos_token": "<s>",
+      "eos_token": "</s>",
+      "tokenizer_class": "OBITokenizer",
+      "unk_token": "</s>",
+      "vocab_size": 10187
 }
+    
+  
+