Upload 2 files

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token":"<mask>}

tokenizer.json

@@ -3,6 +3,33 @@
   "truncation": null,
   "padding": null,
   "added_tokens": [
+    {
+      "id": 0,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
     {
       "id": 3,
       "content": "<unk>",
@@ -11,6 +38,15 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 4,
+      "content": "<mask>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": {
@@ -31,7 +67,7 @@
     "type": "BPE",
     "dropout": null,
     "unk_token": "<unk>",
-    "continuing_subword_prefix": null,
+    "continuing_subword_prefix": "▁",
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "byte_fallback": false,