Upload tokenizer
Browse files
morph_tokenizer/tokenizer_config.json
ADDED
@@ -0,0 +1,698 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"morph_encodings": {
|
3 |
+
"<pad>": 0,
|
4 |
+
"<eos>": 1,
|
5 |
+
"<unk>": 2,
|
6 |
+
"<extra_id_0>": 3,
|
7 |
+
"Adj": 4,
|
8 |
+
"Adj-AFP": 5,
|
9 |
+
"Adj-AFP-C": 6,
|
10 |
+
"Adj-AFS": 7,
|
11 |
+
"Adj-AFS-C": 8,
|
12 |
+
"Adj-AFS-S": 9,
|
13 |
+
"Adj-AMP": 10,
|
14 |
+
"Adj-AMP-C": 11,
|
15 |
+
"Adj-AMS": 12,
|
16 |
+
"Adj-AMS-C": 13,
|
17 |
+
"Adj-ANP": 14,
|
18 |
+
"Adj-ANP-C": 15,
|
19 |
+
"Adj-ANP-S": 16,
|
20 |
+
"Adj-ANS": 17,
|
21 |
+
"Adj-ANS-C": 18,
|
22 |
+
"Adj-ANS-S": 19,
|
23 |
+
"Adj-DFP": 20,
|
24 |
+
"Adj-DFP-C": 21,
|
25 |
+
"Adj-DFS": 22,
|
26 |
+
"Adj-DFS-C": 23,
|
27 |
+
"Adj-DFS-S": 24,
|
28 |
+
"Adj-DMP": 25,
|
29 |
+
"Adj-DMP-C": 26,
|
30 |
+
"Adj-DMS": 27,
|
31 |
+
"Adj-DMS-C": 28,
|
32 |
+
"Adj-DMS-S": 29,
|
33 |
+
"Adj-DNP": 30,
|
34 |
+
"Adj-DNP-S": 31,
|
35 |
+
"Adj-DNS": 32,
|
36 |
+
"Adj-DNS-C": 33,
|
37 |
+
"Adj-DNS-S": 34,
|
38 |
+
"Adj-GFP": 35,
|
39 |
+
"Adj-GFP-S": 36,
|
40 |
+
"Adj-GFS": 37,
|
41 |
+
"Adj-GFS-C": 38,
|
42 |
+
"Adj-GMP": 39,
|
43 |
+
"Adj-GMP-C": 40,
|
44 |
+
"Adj-GMP-S": 41,
|
45 |
+
"Adj-GMS": 42,
|
46 |
+
"Adj-GMS-C": 43,
|
47 |
+
"Adj-GMS-S": 44,
|
48 |
+
"Adj-GNP": 45,
|
49 |
+
"Adj-GNP-C": 46,
|
50 |
+
"Adj-GNP-S": 47,
|
51 |
+
"Adj-GNS": 48,
|
52 |
+
"Adj-GNS-S": 49,
|
53 |
+
"Adj-NFP": 50,
|
54 |
+
"Adj-NFP-C": 51,
|
55 |
+
"Adj-NFP-S": 52,
|
56 |
+
"Adj-NFS": 53,
|
57 |
+
"Adj-NFS-C": 54,
|
58 |
+
"Adj-NFS-S": 55,
|
59 |
+
"Adj-NMP": 56,
|
60 |
+
"Adj-NMP-C": 57,
|
61 |
+
"Adj-NMS": 58,
|
62 |
+
"Adj-NMS-C": 59,
|
63 |
+
"Adj-NMS-S": 60,
|
64 |
+
"Adj-NNP": 61,
|
65 |
+
"Adj-NNP-C": 62,
|
66 |
+
"Adj-NNS": 63,
|
67 |
+
"Adj-NNS-C": 64,
|
68 |
+
"Adj-VFS": 65,
|
69 |
+
"Adj-VMP": 66,
|
70 |
+
"Adj-VMP-C": 67,
|
71 |
+
"Adj-VMS": 68,
|
72 |
+
"Adj-VMS-S": 69,
|
73 |
+
"Adj-VNP": 70,
|
74 |
+
"Adj-VNS": 71,
|
75 |
+
"Adv": 72,
|
76 |
+
"Adv-C": 73,
|
77 |
+
"Adv-S": 74,
|
78 |
+
"Art-AFP": 75,
|
79 |
+
"Art-AFS": 76,
|
80 |
+
"Art-AMP": 77,
|
81 |
+
"Art-AMS": 78,
|
82 |
+
"Art-ANP": 79,
|
83 |
+
"Art-ANS": 80,
|
84 |
+
"Art-DFP": 81,
|
85 |
+
"Art-DFS": 82,
|
86 |
+
"Art-DMP": 83,
|
87 |
+
"Art-DMS": 84,
|
88 |
+
"Art-DNP": 85,
|
89 |
+
"Art-DNS": 86,
|
90 |
+
"Art-GFP": 87,
|
91 |
+
"Art-GFS": 88,
|
92 |
+
"Art-GMP": 89,
|
93 |
+
"Art-GMS": 90,
|
94 |
+
"Art-GNP": 91,
|
95 |
+
"Art-GNS": 92,
|
96 |
+
"Art-NFP": 93,
|
97 |
+
"Art-NFS": 94,
|
98 |
+
"Art-NMP": 95,
|
99 |
+
"Art-NMS": 96,
|
100 |
+
"Art-NNP": 97,
|
101 |
+
"Art-NNS": 98,
|
102 |
+
"Art-VFP": 99,
|
103 |
+
"Art-VFS": 100,
|
104 |
+
"Art-VMP": 101,
|
105 |
+
"Art-VMS": 102,
|
106 |
+
"Art-VNP": 103,
|
107 |
+
"Art-VNS": 104,
|
108 |
+
"Conj": 105,
|
109 |
+
"DPro-AFP": 106,
|
110 |
+
"DPro-AFS": 107,
|
111 |
+
"DPro-AMP": 108,
|
112 |
+
"DPro-AMS": 109,
|
113 |
+
"DPro-ANP": 110,
|
114 |
+
"DPro-ANS": 111,
|
115 |
+
"DPro-DFP": 112,
|
116 |
+
"DPro-DFS": 113,
|
117 |
+
"DPro-DMP": 114,
|
118 |
+
"DPro-DMS": 115,
|
119 |
+
"DPro-DNP": 116,
|
120 |
+
"DPro-DNS": 117,
|
121 |
+
"DPro-GFP": 118,
|
122 |
+
"DPro-GFS": 119,
|
123 |
+
"DPro-GMP": 120,
|
124 |
+
"DPro-GMS": 121,
|
125 |
+
"DPro-GNP": 122,
|
126 |
+
"DPro-GNS": 123,
|
127 |
+
"DPro-NFP": 124,
|
128 |
+
"DPro-NFS": 125,
|
129 |
+
"DPro-NMP": 126,
|
130 |
+
"DPro-NMS": 127,
|
131 |
+
"DPro-NNP": 128,
|
132 |
+
"DPro-NNS": 129,
|
133 |
+
"Heb": 130,
|
134 |
+
"I": 131,
|
135 |
+
"IPro-AFP": 132,
|
136 |
+
"IPro-AFS": 133,
|
137 |
+
"IPro-AMP": 134,
|
138 |
+
"IPro-AMS": 135,
|
139 |
+
"IPro-ANP": 136,
|
140 |
+
"IPro-ANS": 137,
|
141 |
+
"IPro-DFS": 138,
|
142 |
+
"IPro-DMP": 139,
|
143 |
+
"IPro-DMS": 140,
|
144 |
+
"IPro-DNS": 141,
|
145 |
+
"IPro-GFP": 142,
|
146 |
+
"IPro-GFS": 143,
|
147 |
+
"IPro-GMP": 144,
|
148 |
+
"IPro-GMS": 145,
|
149 |
+
"IPro-GNP": 146,
|
150 |
+
"IPro-GNS": 147,
|
151 |
+
"IPro-NFP": 148,
|
152 |
+
"IPro-NFS": 149,
|
153 |
+
"IPro-NMP": 150,
|
154 |
+
"IPro-NMS": 151,
|
155 |
+
"IPro-NNP": 152,
|
156 |
+
"IPro-NNS": 153,
|
157 |
+
"Indec": 154,
|
158 |
+
"IntPrtcl": 155,
|
159 |
+
"N": 156,
|
160 |
+
"N-AFP": 157,
|
161 |
+
"N-AFS": 158,
|
162 |
+
"N-AMP": 159,
|
163 |
+
"N-AMS": 160,
|
164 |
+
"N-ANP": 161,
|
165 |
+
"N-ANS": 162,
|
166 |
+
"N-DFP": 163,
|
167 |
+
"N-DFS": 164,
|
168 |
+
"N-DMP": 165,
|
169 |
+
"N-DMS": 166,
|
170 |
+
"N-DNP": 167,
|
171 |
+
"N-DNS": 168,
|
172 |
+
"N-GFP": 169,
|
173 |
+
"N-GFS": 170,
|
174 |
+
"N-GMP": 171,
|
175 |
+
"N-GMS": 172,
|
176 |
+
"N-GNP": 173,
|
177 |
+
"N-GNS": 174,
|
178 |
+
"N-NFP": 175,
|
179 |
+
"N-NFS": 176,
|
180 |
+
"N-NMP": 177,
|
181 |
+
"N-NMS": 178,
|
182 |
+
"N-NNP": 179,
|
183 |
+
"N-NNS": 180,
|
184 |
+
"N-VFP": 181,
|
185 |
+
"N-VFS": 182,
|
186 |
+
"N-VMP": 183,
|
187 |
+
"N-VMS": 184,
|
188 |
+
"N-VNP": 185,
|
189 |
+
"N-VNS": 186,
|
190 |
+
"PPro-A1P": 187,
|
191 |
+
"PPro-A1S": 188,
|
192 |
+
"PPro-A2P": 189,
|
193 |
+
"PPro-A2S": 190,
|
194 |
+
"PPro-AF1P": 191,
|
195 |
+
"PPro-AF1S": 192,
|
196 |
+
"PPro-AF2P": 193,
|
197 |
+
"PPro-AF2S": 194,
|
198 |
+
"PPro-AF3P": 195,
|
199 |
+
"PPro-AF3S": 196,
|
200 |
+
"PPro-AM1P": 197,
|
201 |
+
"PPro-AM1S": 198,
|
202 |
+
"PPro-AM2P": 199,
|
203 |
+
"PPro-AM2S": 200,
|
204 |
+
"PPro-AM3P": 201,
|
205 |
+
"PPro-AM3S": 202,
|
206 |
+
"PPro-AN1P": 203,
|
207 |
+
"PPro-AN1S": 204,
|
208 |
+
"PPro-AN2P": 205,
|
209 |
+
"PPro-AN2S": 206,
|
210 |
+
"PPro-AN3P": 207,
|
211 |
+
"PPro-AN3S": 208,
|
212 |
+
"PPro-D1P": 209,
|
213 |
+
"PPro-D1S": 210,
|
214 |
+
"PPro-D2P": 211,
|
215 |
+
"PPro-D2S": 212,
|
216 |
+
"PPro-DF1P": 213,
|
217 |
+
"PPro-DF1S": 214,
|
218 |
+
"PPro-DF2P": 215,
|
219 |
+
"PPro-DF2S": 216,
|
220 |
+
"PPro-DF3P": 217,
|
221 |
+
"PPro-DF3S": 218,
|
222 |
+
"PPro-DM1P": 219,
|
223 |
+
"PPro-DM1S": 220,
|
224 |
+
"PPro-DM2P": 221,
|
225 |
+
"PPro-DM2S": 222,
|
226 |
+
"PPro-DM3P": 223,
|
227 |
+
"PPro-DM3S": 224,
|
228 |
+
"PPro-DN1P": 225,
|
229 |
+
"PPro-DN1S": 226,
|
230 |
+
"PPro-DN3P": 227,
|
231 |
+
"PPro-DN3S": 228,
|
232 |
+
"PPro-G1P": 229,
|
233 |
+
"PPro-G1S": 230,
|
234 |
+
"PPro-G2P": 231,
|
235 |
+
"PPro-G2S": 232,
|
236 |
+
"PPro-GF1P": 233,
|
237 |
+
"PPro-GF1S": 234,
|
238 |
+
"PPro-GF2P": 235,
|
239 |
+
"PPro-GF2S": 236,
|
240 |
+
"PPro-GF3P": 237,
|
241 |
+
"PPro-GF3S": 238,
|
242 |
+
"PPro-GM1S": 239,
|
243 |
+
"PPro-GM2S": 240,
|
244 |
+
"PPro-GM3P": 241,
|
245 |
+
"PPro-GM3S": 242,
|
246 |
+
"PPro-GN1P": 243,
|
247 |
+
"PPro-GN3P": 244,
|
248 |
+
"PPro-GN3S": 245,
|
249 |
+
"PPro-N1P": 246,
|
250 |
+
"PPro-N1S": 247,
|
251 |
+
"PPro-N2P": 248,
|
252 |
+
"PPro-N2S": 249,
|
253 |
+
"PPro-NF1P": 250,
|
254 |
+
"PPro-NF1S": 251,
|
255 |
+
"PPro-NF2P": 252,
|
256 |
+
"PPro-NF3S": 253,
|
257 |
+
"PPro-NFS": 254,
|
258 |
+
"PPro-NM1P": 255,
|
259 |
+
"PPro-NM1S": 256,
|
260 |
+
"PPro-NM2P": 257,
|
261 |
+
"PPro-NM2S": 258,
|
262 |
+
"PPro-NM3P": 259,
|
263 |
+
"PPro-NM3S": 260,
|
264 |
+
"PPro-NN1P": 261,
|
265 |
+
"PPro-NN1S": 262,
|
266 |
+
"PPro-NN2P": 263,
|
267 |
+
"PPro-NN2S": 264,
|
268 |
+
"PPro-NN3P": 265,
|
269 |
+
"PPro-NN3S": 266,
|
270 |
+
"Prep": 267,
|
271 |
+
"Prtcl": 268,
|
272 |
+
"RecPro-AMP": 269,
|
273 |
+
"RecPro-DMP": 270,
|
274 |
+
"RecPro-DNP": 271,
|
275 |
+
"RecPro-GMP": 272,
|
276 |
+
"RefPro-AF3P": 273,
|
277 |
+
"RefPro-AF3S": 274,
|
278 |
+
"RefPro-AM3P": 275,
|
279 |
+
"RefPro-AM3S": 276,
|
280 |
+
"RefPro-AN3P": 277,
|
281 |
+
"RefPro-DF3P": 278,
|
282 |
+
"RefPro-DF3S": 279,
|
283 |
+
"RefPro-DM3P": 280,
|
284 |
+
"RefPro-DM3S": 281,
|
285 |
+
"RefPro-GF3P": 282,
|
286 |
+
"RefPro-GF3S": 283,
|
287 |
+
"RefPro-GM3P": 284,
|
288 |
+
"RefPro-GM3S": 285,
|
289 |
+
"RefPro-GN3P": 286,
|
290 |
+
"RefPro-GN3S": 287,
|
291 |
+
"RelPro-AFP": 288,
|
292 |
+
"RelPro-AFS": 289,
|
293 |
+
"RelPro-AMP": 290,
|
294 |
+
"RelPro-AMS": 291,
|
295 |
+
"RelPro-ANP": 292,
|
296 |
+
"RelPro-ANS": 293,
|
297 |
+
"RelPro-DFP": 294,
|
298 |
+
"RelPro-DFS": 295,
|
299 |
+
"RelPro-DMP": 296,
|
300 |
+
"RelPro-DMS": 297,
|
301 |
+
"RelPro-DNP": 298,
|
302 |
+
"RelPro-DNS": 299,
|
303 |
+
"RelPro-GFP": 300,
|
304 |
+
"RelPro-GFS": 301,
|
305 |
+
"RelPro-GMP": 302,
|
306 |
+
"RelPro-GMS": 303,
|
307 |
+
"RelPro-GNP": 304,
|
308 |
+
"RelPro-GNS": 305,
|
309 |
+
"RelPro-NFP": 306,
|
310 |
+
"RelPro-NFS": 307,
|
311 |
+
"RelPro-NMP": 308,
|
312 |
+
"RelPro-NMS": 309,
|
313 |
+
"RelPro-NNP": 310,
|
314 |
+
"RelPro-NNS": 311,
|
315 |
+
"V-AIA-1P": 312,
|
316 |
+
"V-AIA-1S": 313,
|
317 |
+
"V-AIA-2P": 314,
|
318 |
+
"V-AIA-2S": 315,
|
319 |
+
"V-AIA-3P": 316,
|
320 |
+
"V-AIA-3S": 317,
|
321 |
+
"V-AIM-1P": 318,
|
322 |
+
"V-AIM-1S": 319,
|
323 |
+
"V-AIM-2P": 320,
|
324 |
+
"V-AIM-2S": 321,
|
325 |
+
"V-AIM-3P": 322,
|
326 |
+
"V-AIM-3S": 323,
|
327 |
+
"V-AIP-1P": 324,
|
328 |
+
"V-AIP-1S": 325,
|
329 |
+
"V-AIP-2P": 326,
|
330 |
+
"V-AIP-2S": 327,
|
331 |
+
"V-AIP-3P": 328,
|
332 |
+
"V-AIP-3S": 329,
|
333 |
+
"V-AMA-2P": 330,
|
334 |
+
"V-AMA-2S": 331,
|
335 |
+
"V-AMA-3P": 332,
|
336 |
+
"V-AMA-3S": 333,
|
337 |
+
"V-AMM-2P": 334,
|
338 |
+
"V-AMM-2S": 335,
|
339 |
+
"V-AMM-3P": 336,
|
340 |
+
"V-AMM-3S": 337,
|
341 |
+
"V-AMP-2P": 338,
|
342 |
+
"V-AMP-2S": 339,
|
343 |
+
"V-AMP-3P": 340,
|
344 |
+
"V-AMP-3S": 341,
|
345 |
+
"V-ANA": 342,
|
346 |
+
"V-ANM": 343,
|
347 |
+
"V-ANM/P": 344,
|
348 |
+
"V-ANP": 345,
|
349 |
+
"V-AOA-3P": 346,
|
350 |
+
"V-AOA-3S": 347,
|
351 |
+
"V-AOM-1S": 348,
|
352 |
+
"V-AOM-3S": 349,
|
353 |
+
"V-AOP-3S": 350,
|
354 |
+
"V-APA-AFP": 351,
|
355 |
+
"V-APA-AFS": 352,
|
356 |
+
"V-APA-AMP": 353,
|
357 |
+
"V-APA-AMS": 354,
|
358 |
+
"V-APA-ANP": 355,
|
359 |
+
"V-APA-ANS": 356,
|
360 |
+
"V-APA-DFP": 357,
|
361 |
+
"V-APA-DFS": 358,
|
362 |
+
"V-APA-DMP": 359,
|
363 |
+
"V-APA-DMS": 360,
|
364 |
+
"V-APA-GFS": 361,
|
365 |
+
"V-APA-GMP": 362,
|
366 |
+
"V-APA-GMS": 363,
|
367 |
+
"V-APA-GNS": 364,
|
368 |
+
"V-APA-NFP": 365,
|
369 |
+
"V-APA-NFS": 366,
|
370 |
+
"V-APA-NMP": 367,
|
371 |
+
"V-APA-NMS": 368,
|
372 |
+
"V-APA-NNP": 369,
|
373 |
+
"V-APA-NNS": 370,
|
374 |
+
"V-APM-AFS": 371,
|
375 |
+
"V-APM-AMP": 372,
|
376 |
+
"V-APM-AMS": 373,
|
377 |
+
"V-APM-ANP": 374,
|
378 |
+
"V-APM-ANS": 375,
|
379 |
+
"V-APM-DMP": 376,
|
380 |
+
"V-APM-DNP": 377,
|
381 |
+
"V-APM-GFP": 378,
|
382 |
+
"V-APM-GFS": 379,
|
383 |
+
"V-APM-GMP": 380,
|
384 |
+
"V-APM-GMS": 381,
|
385 |
+
"V-APM-GNP": 382,
|
386 |
+
"V-APM-GNS": 383,
|
387 |
+
"V-APM-NFP": 384,
|
388 |
+
"V-APM-NFS": 385,
|
389 |
+
"V-APM-NMP": 386,
|
390 |
+
"V-APM-NMS": 387,
|
391 |
+
"V-APM-NNS": 388,
|
392 |
+
"V-APM/P-ANP": 389,
|
393 |
+
"V-APM/P-GFS": 390,
|
394 |
+
"V-APM/P-NMS": 391,
|
395 |
+
"V-APP-AFS": 392,
|
396 |
+
"V-APP-AMP": 393,
|
397 |
+
"V-APP-AMS": 394,
|
398 |
+
"V-APP-ANP": 395,
|
399 |
+
"V-APP-ANS": 396,
|
400 |
+
"V-APP-DFS": 397,
|
401 |
+
"V-APP-DMS": 398,
|
402 |
+
"V-APP-DNP": 399,
|
403 |
+
"V-APP-DNS": 400,
|
404 |
+
"V-APP-GFP": 401,
|
405 |
+
"V-APP-GFS": 402,
|
406 |
+
"V-APP-GMP": 403,
|
407 |
+
"V-APP-GMS": 404,
|
408 |
+
"V-APP-GNP": 405,
|
409 |
+
"V-APP-GNS": 406,
|
410 |
+
"V-APP-NFP": 407,
|
411 |
+
"V-APP-NFS": 408,
|
412 |
+
"V-APP-NMP": 409,
|
413 |
+
"V-APP-NMS": 410,
|
414 |
+
"V-APP-NNP": 411,
|
415 |
+
"V-APP-NNS": 412,
|
416 |
+
"V-ASA-1P": 413,
|
417 |
+
"V-ASA-1S": 414,
|
418 |
+
"V-ASA-2P": 415,
|
419 |
+
"V-ASA-2S": 416,
|
420 |
+
"V-ASA-3P": 417,
|
421 |
+
"V-ASA-3S": 418,
|
422 |
+
"V-ASM-1P": 419,
|
423 |
+
"V-ASM-1S": 420,
|
424 |
+
"V-ASM-2P": 421,
|
425 |
+
"V-ASM-2S": 422,
|
426 |
+
"V-ASM-3P": 423,
|
427 |
+
"V-ASM-3S": 424,
|
428 |
+
"V-ASP-1P": 425,
|
429 |
+
"V-ASP-1S": 426,
|
430 |
+
"V-ASP-2P": 427,
|
431 |
+
"V-ASP-2S": 428,
|
432 |
+
"V-ASP-3P": 429,
|
433 |
+
"V-ASP-3S": 430,
|
434 |
+
"V-FIA-1P": 431,
|
435 |
+
"V-FIA-1S": 432,
|
436 |
+
"V-FIA-2P": 433,
|
437 |
+
"V-FIA-2S": 434,
|
438 |
+
"V-FIA-3P": 435,
|
439 |
+
"V-FIA-3S": 436,
|
440 |
+
"V-FIM-1P": 437,
|
441 |
+
"V-FIM-1S": 438,
|
442 |
+
"V-FIM-2P": 439,
|
443 |
+
"V-FIM-2S": 440,
|
444 |
+
"V-FIM-3P": 441,
|
445 |
+
"V-FIM-3S": 442,
|
446 |
+
"V-FIM/P-2P": 443,
|
447 |
+
"V-FIP-1P": 444,
|
448 |
+
"V-FIP-1S": 445,
|
449 |
+
"V-FIP-2P": 446,
|
450 |
+
"V-FIP-2S": 447,
|
451 |
+
"V-FIP-3P": 448,
|
452 |
+
"V-FIP-3S": 449,
|
453 |
+
"V-FNM": 450,
|
454 |
+
"V-FPA-ANP": 451,
|
455 |
+
"V-FPA-NMP": 452,
|
456 |
+
"V-FPA-NMS": 453,
|
457 |
+
"V-FPM-ANS": 454,
|
458 |
+
"V-FPP-GNP": 455,
|
459 |
+
"V-IIA-1P": 456,
|
460 |
+
"V-IIA-1S": 457,
|
461 |
+
"V-IIA-2P": 458,
|
462 |
+
"V-IIA-2S": 459,
|
463 |
+
"V-IIA-3P": 460,
|
464 |
+
"V-IIA-3S": 461,
|
465 |
+
"V-IIM-1P": 462,
|
466 |
+
"V-IIM-1S": 463,
|
467 |
+
"V-IIM-2S": 464,
|
468 |
+
"V-IIM-3P": 465,
|
469 |
+
"V-IIM-3S": 466,
|
470 |
+
"V-IIM/P-1P": 467,
|
471 |
+
"V-IIM/P-1S": 468,
|
472 |
+
"V-IIM/P-2P": 469,
|
473 |
+
"V-IIM/P-2S": 470,
|
474 |
+
"V-IIM/P-3P": 471,
|
475 |
+
"V-IIM/P-3S": 472,
|
476 |
+
"V-IIP-1P": 473,
|
477 |
+
"V-IIP-3P": 474,
|
478 |
+
"V-LIA-1S": 475,
|
479 |
+
"V-LIA-2P": 476,
|
480 |
+
"V-LIA-2S": 477,
|
481 |
+
"V-LIA-3P": 478,
|
482 |
+
"V-LIA-3S": 479,
|
483 |
+
"V-LIM-3P": 480,
|
484 |
+
"V-LIM-3S": 481,
|
485 |
+
"V-LIM/P-3S": 482,
|
486 |
+
"V-M-2P": 483,
|
487 |
+
"V-M-2S": 484,
|
488 |
+
"V-PIA-1P": 485,
|
489 |
+
"V-PIA-1S": 486,
|
490 |
+
"V-PIA-2P": 487,
|
491 |
+
"V-PIA-2S": 488,
|
492 |
+
"V-PIA-3P": 489,
|
493 |
+
"V-PIA-3S": 490,
|
494 |
+
"V-PIM-1P": 491,
|
495 |
+
"V-PIM-1S": 492,
|
496 |
+
"V-PIM-2P": 493,
|
497 |
+
"V-PIM-3P": 494,
|
498 |
+
"V-PIM-3S": 495,
|
499 |
+
"V-PIM/P-1P": 496,
|
500 |
+
"V-PIM/P-1S": 497,
|
501 |
+
"V-PIM/P-2P": 498,
|
502 |
+
"V-PIM/P-2S": 499,
|
503 |
+
"V-PIM/P-3P": 500,
|
504 |
+
"V-PIM/P-3S": 501,
|
505 |
+
"V-PIP-1S": 502,
|
506 |
+
"V-PIP-3S": 503,
|
507 |
+
"V-PMA-2P": 504,
|
508 |
+
"V-PMA-2S": 505,
|
509 |
+
"V-PMA-3P": 506,
|
510 |
+
"V-PMA-3S": 507,
|
511 |
+
"V-PMM-2P": 508,
|
512 |
+
"V-PMM-2S": 509,
|
513 |
+
"V-PMM/P-2P": 510,
|
514 |
+
"V-PMM/P-2S": 511,
|
515 |
+
"V-PMM/P-3P": 512,
|
516 |
+
"V-PMM/P-3S": 513,
|
517 |
+
"V-PMP-2P": 514,
|
518 |
+
"V-PMP-3S": 515,
|
519 |
+
"V-PNA": 516,
|
520 |
+
"V-PNM": 517,
|
521 |
+
"V-PNM/P": 518,
|
522 |
+
"V-PNP": 519,
|
523 |
+
"V-POA-2P": 520,
|
524 |
+
"V-POA-3P": 521,
|
525 |
+
"V-POA-3S": 522,
|
526 |
+
"V-POM/P-1S": 523,
|
527 |
+
"V-POM/P-3P": 524,
|
528 |
+
"V-POM/P-3S": 525,
|
529 |
+
"V-PPA-AFP": 526,
|
530 |
+
"V-PPA-AFS": 527,
|
531 |
+
"V-PPA-AMP": 528,
|
532 |
+
"V-PPA-AMS": 529,
|
533 |
+
"V-PPA-ANP": 530,
|
534 |
+
"V-PPA-ANS": 531,
|
535 |
+
"V-PPA-DFP": 532,
|
536 |
+
"V-PPA-DFS": 533,
|
537 |
+
"V-PPA-DMP": 534,
|
538 |
+
"V-PPA-DMS": 535,
|
539 |
+
"V-PPA-DNP": 536,
|
540 |
+
"V-PPA-DNS": 537,
|
541 |
+
"V-PPA-GFP": 538,
|
542 |
+
"V-PPA-GFS": 539,
|
543 |
+
"V-PPA-GMP": 540,
|
544 |
+
"V-PPA-GMS": 541,
|
545 |
+
"V-PPA-GNP": 542,
|
546 |
+
"V-PPA-GNS": 543,
|
547 |
+
"V-PPA-NFP": 544,
|
548 |
+
"V-PPA-NFS": 545,
|
549 |
+
"V-PPA-NMP": 546,
|
550 |
+
"V-PPA-NMS": 547,
|
551 |
+
"V-PPA-NNP": 548,
|
552 |
+
"V-PPA-NNS": 549,
|
553 |
+
"V-PPA-VFS": 550,
|
554 |
+
"V-PPA-VMP": 551,
|
555 |
+
"V-PPA-VMS": 552,
|
556 |
+
"V-PPM-AFS": 553,
|
557 |
+
"V-PPM-AMP": 554,
|
558 |
+
"V-PPM-AMS": 555,
|
559 |
+
"V-PPM-ANS": 556,
|
560 |
+
"V-PPM-DMP": 557,
|
561 |
+
"V-PPM-DMS": 558,
|
562 |
+
"V-PPM-GFP": 559,
|
563 |
+
"V-PPM-GFS": 560,
|
564 |
+
"V-PPM-GMP": 561,
|
565 |
+
"V-PPM-GMS": 562,
|
566 |
+
"V-PPM-GNP": 563,
|
567 |
+
"V-PPM-GNS": 564,
|
568 |
+
"V-PPM-NFP": 565,
|
569 |
+
"V-PPM-NFS": 566,
|
570 |
+
"V-PPM-NMP": 567,
|
571 |
+
"V-PPM-NMS": 568,
|
572 |
+
"V-PPM-NNS": 569,
|
573 |
+
"V-PPM/P-AFP": 570,
|
574 |
+
"V-PPM/P-AFS": 571,
|
575 |
+
"V-PPM/P-AMP": 572,
|
576 |
+
"V-PPM/P-AMS": 573,
|
577 |
+
"V-PPM/P-ANP": 574,
|
578 |
+
"V-PPM/P-ANS": 575,
|
579 |
+
"V-PPM/P-DFP": 576,
|
580 |
+
"V-PPM/P-DFS": 577,
|
581 |
+
"V-PPM/P-DMP": 578,
|
582 |
+
"V-PPM/P-DMS": 579,
|
583 |
+
"V-PPM/P-DNP": 580,
|
584 |
+
"V-PPM/P-DNS": 581,
|
585 |
+
"V-PPM/P-GFP": 582,
|
586 |
+
"V-PPM/P-GFS": 583,
|
587 |
+
"V-PPM/P-GMP": 584,
|
588 |
+
"V-PPM/P-GMS": 585,
|
589 |
+
"V-PPM/P-GNP": 586,
|
590 |
+
"V-PPM/P-GNS": 587,
|
591 |
+
"V-PPM/P-NFP": 588,
|
592 |
+
"V-PPM/P-NFS": 589,
|
593 |
+
"V-PPM/P-NMP": 590,
|
594 |
+
"V-PPM/P-NMS": 591,
|
595 |
+
"V-PPM/P-NNP": 592,
|
596 |
+
"V-PPM/P-NNS": 593,
|
597 |
+
"V-PPM/P-VMP": 594,
|
598 |
+
"V-PPM/P-VMS": 595,
|
599 |
+
"V-PPP-DMP": 596,
|
600 |
+
"V-PPP-GMP": 597,
|
601 |
+
"V-PPP-GMS": 598,
|
602 |
+
"V-PPP-NMP": 599,
|
603 |
+
"V-PSA-1P": 600,
|
604 |
+
"V-PSA-1S": 601,
|
605 |
+
"V-PSA-2P": 602,
|
606 |
+
"V-PSA-2S": 603,
|
607 |
+
"V-PSA-3P": 604,
|
608 |
+
"V-PSA-3S": 605,
|
609 |
+
"V-PSM-1P": 606,
|
610 |
+
"V-PSM-1S": 607,
|
611 |
+
"V-PSM-2S": 608,
|
612 |
+
"V-PSM-3S": 609,
|
613 |
+
"V-PSM/P-1P": 610,
|
614 |
+
"V-PSM/P-1S": 611,
|
615 |
+
"V-PSM/P-2P": 612,
|
616 |
+
"V-PSM/P-2S": 613,
|
617 |
+
"V-PSM/P-3P": 614,
|
618 |
+
"V-PSM/P-3S": 615,
|
619 |
+
"V-RIA-1P": 616,
|
620 |
+
"V-RIA-1S": 617,
|
621 |
+
"V-RIA-2P": 618,
|
622 |
+
"V-RIA-2S": 619,
|
623 |
+
"V-RIA-3P": 620,
|
624 |
+
"V-RIA-3S": 621,
|
625 |
+
"V-RIM-2S": 622,
|
626 |
+
"V-RIM-3S": 623,
|
627 |
+
"V-RIM/P-1P": 624,
|
628 |
+
"V-RIM/P-1S": 625,
|
629 |
+
"V-RIM/P-2P": 626,
|
630 |
+
"V-RIM/P-2S": 627,
|
631 |
+
"V-RIM/P-3P": 628,
|
632 |
+
"V-RIM/P-3S": 629,
|
633 |
+
"V-RIP-1P": 630,
|
634 |
+
"V-RMA-2P": 631,
|
635 |
+
"V-RMM/P-2P": 632,
|
636 |
+
"V-RMM/P-2S": 633,
|
637 |
+
"V-RNA": 634,
|
638 |
+
"V-RNM/P": 635,
|
639 |
+
"V-RPA-AFS": 636,
|
640 |
+
"V-RPA-AMP": 637,
|
641 |
+
"V-RPA-AMS": 638,
|
642 |
+
"V-RPA-ANP": 639,
|
643 |
+
"V-RPA-ANS": 640,
|
644 |
+
"V-RPA-DMP": 641,
|
645 |
+
"V-RPA-DMS": 642,
|
646 |
+
"V-RPA-DNS": 643,
|
647 |
+
"V-RPA-GFS": 644,
|
648 |
+
"V-RPA-GMP": 645,
|
649 |
+
"V-RPA-GMS": 646,
|
650 |
+
"V-RPA-GNP": 647,
|
651 |
+
"V-RPA-NFP": 648,
|
652 |
+
"V-RPA-NFS": 649,
|
653 |
+
"V-RPA-NMP": 650,
|
654 |
+
"V-RPA-NMS": 651,
|
655 |
+
"V-RPA-NNP": 652,
|
656 |
+
"V-RPA-NNS": 653,
|
657 |
+
"V-RPM-AMS": 654,
|
658 |
+
"V-RPM-NMP": 655,
|
659 |
+
"V-RPM-NMS": 656,
|
660 |
+
"V-RPM/P-AFP": 657,
|
661 |
+
"V-RPM/P-AFS": 658,
|
662 |
+
"V-RPM/P-AMP": 659,
|
663 |
+
"V-RPM/P-AMS": 660,
|
664 |
+
"V-RPM/P-ANP": 661,
|
665 |
+
"V-RPM/P-ANS": 662,
|
666 |
+
"V-RPM/P-DFP": 663,
|
667 |
+
"V-RPM/P-DFS": 664,
|
668 |
+
"V-RPM/P-DMP": 665,
|
669 |
+
"V-RPM/P-DMS": 666,
|
670 |
+
"V-RPM/P-DNP": 667,
|
671 |
+
"V-RPM/P-DNS": 668,
|
672 |
+
"V-RPM/P-GFP": 669,
|
673 |
+
"V-RPM/P-GFS": 670,
|
674 |
+
"V-RPM/P-GMP": 671,
|
675 |
+
"V-RPM/P-GMS": 672,
|
676 |
+
"V-RPM/P-GNP": 673,
|
677 |
+
"V-RPM/P-GNS": 674,
|
678 |
+
"V-RPM/P-NFP": 675,
|
679 |
+
"V-RPM/P-NFS": 676,
|
680 |
+
"V-RPM/P-NMP": 677,
|
681 |
+
"V-RPM/P-NMS": 678,
|
682 |
+
"V-RPM/P-NNP": 679,
|
683 |
+
"V-RPM/P-NNS": 680,
|
684 |
+
"V-RPM/P-VFS": 681,
|
685 |
+
"V-RPM/P-VMP": 682,
|
686 |
+
"V-RPM/P-VMS": 683,
|
687 |
+
"V-RSA-1P": 684,
|
688 |
+
"V-RSA-1S": 685,
|
689 |
+
"V-RSA-2P": 686,
|
690 |
+
"V-RSA-2S": 687
|
691 |
+
},
|
692 |
+
"special_tokens_map": {
|
693 |
+
"pad_token": "<pad>",
|
694 |
+
"eos_token": "<eos>",
|
695 |
+
"unk_token": "<unk>",
|
696 |
+
"block_separator_token": "<extra_id_0>"
|
697 |
+
}
|
698 |
+
}
|
text_tokenizer/special_tokens_map.json
ADDED
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
"<extra_id_0>",
|
4 |
+
"<extra_id_1>",
|
5 |
+
"<extra_id_2>",
|
6 |
+
"<extra_id_3>",
|
7 |
+
"<extra_id_4>",
|
8 |
+
"<extra_id_5>",
|
9 |
+
"<extra_id_6>",
|
10 |
+
"<extra_id_7>",
|
11 |
+
"<extra_id_8>",
|
12 |
+
"<extra_id_9>",
|
13 |
+
"<extra_id_10>",
|
14 |
+
"<extra_id_11>",
|
15 |
+
"<extra_id_12>",
|
16 |
+
"<extra_id_13>",
|
17 |
+
"<extra_id_14>",
|
18 |
+
"<extra_id_15>",
|
19 |
+
"<extra_id_16>",
|
20 |
+
"<extra_id_17>",
|
21 |
+
"<extra_id_18>",
|
22 |
+
"<extra_id_19>",
|
23 |
+
"<extra_id_20>",
|
24 |
+
"<extra_id_21>",
|
25 |
+
"<extra_id_22>",
|
26 |
+
"<extra_id_23>",
|
27 |
+
"<extra_id_24>",
|
28 |
+
"<extra_id_25>",
|
29 |
+
"<extra_id_26>",
|
30 |
+
"<extra_id_27>",
|
31 |
+
"<extra_id_28>",
|
32 |
+
"<extra_id_29>",
|
33 |
+
"<extra_id_30>",
|
34 |
+
"<extra_id_31>",
|
35 |
+
"<extra_id_32>",
|
36 |
+
"<extra_id_33>",
|
37 |
+
"<extra_id_34>",
|
38 |
+
"<extra_id_35>",
|
39 |
+
"<extra_id_36>",
|
40 |
+
"<extra_id_37>",
|
41 |
+
"<extra_id_38>",
|
42 |
+
"<extra_id_39>",
|
43 |
+
"<extra_id_40>",
|
44 |
+
"<extra_id_41>",
|
45 |
+
"<extra_id_42>",
|
46 |
+
"<extra_id_43>",
|
47 |
+
"<extra_id_44>",
|
48 |
+
"<extra_id_45>",
|
49 |
+
"<extra_id_46>",
|
50 |
+
"<extra_id_47>",
|
51 |
+
"<extra_id_48>",
|
52 |
+
"<extra_id_49>",
|
53 |
+
"<extra_id_50>",
|
54 |
+
"<extra_id_51>",
|
55 |
+
"<extra_id_52>",
|
56 |
+
"<extra_id_53>",
|
57 |
+
"<extra_id_54>",
|
58 |
+
"<extra_id_55>",
|
59 |
+
"<extra_id_56>",
|
60 |
+
"<extra_id_57>",
|
61 |
+
"<extra_id_58>",
|
62 |
+
"<extra_id_59>",
|
63 |
+
"<extra_id_60>",
|
64 |
+
"<extra_id_61>",
|
65 |
+
"<extra_id_62>",
|
66 |
+
"<extra_id_63>",
|
67 |
+
"<extra_id_64>",
|
68 |
+
"<extra_id_65>",
|
69 |
+
"<extra_id_66>",
|
70 |
+
"<extra_id_67>",
|
71 |
+
"<extra_id_68>",
|
72 |
+
"<extra_id_69>",
|
73 |
+
"<extra_id_70>",
|
74 |
+
"<extra_id_71>",
|
75 |
+
"<extra_id_72>",
|
76 |
+
"<extra_id_73>",
|
77 |
+
"<extra_id_74>",
|
78 |
+
"<extra_id_75>",
|
79 |
+
"<extra_id_76>",
|
80 |
+
"<extra_id_77>",
|
81 |
+
"<extra_id_78>",
|
82 |
+
"<extra_id_79>",
|
83 |
+
"<extra_id_80>",
|
84 |
+
"<extra_id_81>",
|
85 |
+
"<extra_id_82>",
|
86 |
+
"<extra_id_83>",
|
87 |
+
"<extra_id_84>",
|
88 |
+
"<extra_id_85>",
|
89 |
+
"<extra_id_86>",
|
90 |
+
"<extra_id_87>",
|
91 |
+
"<extra_id_88>",
|
92 |
+
"<extra_id_89>",
|
93 |
+
"<extra_id_90>",
|
94 |
+
"<extra_id_91>",
|
95 |
+
"<extra_id_92>",
|
96 |
+
"<extra_id_93>",
|
97 |
+
"<extra_id_94>",
|
98 |
+
"<extra_id_95>",
|
99 |
+
"<extra_id_96>",
|
100 |
+
"<extra_id_97>",
|
101 |
+
"<extra_id_98>",
|
102 |
+
"<extra_id_99>"
|
103 |
+
],
|
104 |
+
"eos_token": "</s>",
|
105 |
+
"pad_token": "<pad>",
|
106 |
+
"unk_token": "<unk>"
|
107 |
+
}
|
text_tokenizer/tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
text_tokenizer/tokenizer_config.json
ADDED
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
"<extra_id_0>",
|
4 |
+
"<extra_id_1>",
|
5 |
+
"<extra_id_2>",
|
6 |
+
"<extra_id_3>",
|
7 |
+
"<extra_id_4>",
|
8 |
+
"<extra_id_5>",
|
9 |
+
"<extra_id_6>",
|
10 |
+
"<extra_id_7>",
|
11 |
+
"<extra_id_8>",
|
12 |
+
"<extra_id_9>",
|
13 |
+
"<extra_id_10>",
|
14 |
+
"<extra_id_11>",
|
15 |
+
"<extra_id_12>",
|
16 |
+
"<extra_id_13>",
|
17 |
+
"<extra_id_14>",
|
18 |
+
"<extra_id_15>",
|
19 |
+
"<extra_id_16>",
|
20 |
+
"<extra_id_17>",
|
21 |
+
"<extra_id_18>",
|
22 |
+
"<extra_id_19>",
|
23 |
+
"<extra_id_20>",
|
24 |
+
"<extra_id_21>",
|
25 |
+
"<extra_id_22>",
|
26 |
+
"<extra_id_23>",
|
27 |
+
"<extra_id_24>",
|
28 |
+
"<extra_id_25>",
|
29 |
+
"<extra_id_26>",
|
30 |
+
"<extra_id_27>",
|
31 |
+
"<extra_id_28>",
|
32 |
+
"<extra_id_29>",
|
33 |
+
"<extra_id_30>",
|
34 |
+
"<extra_id_31>",
|
35 |
+
"<extra_id_32>",
|
36 |
+
"<extra_id_33>",
|
37 |
+
"<extra_id_34>",
|
38 |
+
"<extra_id_35>",
|
39 |
+
"<extra_id_36>",
|
40 |
+
"<extra_id_37>",
|
41 |
+
"<extra_id_38>",
|
42 |
+
"<extra_id_39>",
|
43 |
+
"<extra_id_40>",
|
44 |
+
"<extra_id_41>",
|
45 |
+
"<extra_id_42>",
|
46 |
+
"<extra_id_43>",
|
47 |
+
"<extra_id_44>",
|
48 |
+
"<extra_id_45>",
|
49 |
+
"<extra_id_46>",
|
50 |
+
"<extra_id_47>",
|
51 |
+
"<extra_id_48>",
|
52 |
+
"<extra_id_49>",
|
53 |
+
"<extra_id_50>",
|
54 |
+
"<extra_id_51>",
|
55 |
+
"<extra_id_52>",
|
56 |
+
"<extra_id_53>",
|
57 |
+
"<extra_id_54>",
|
58 |
+
"<extra_id_55>",
|
59 |
+
"<extra_id_56>",
|
60 |
+
"<extra_id_57>",
|
61 |
+
"<extra_id_58>",
|
62 |
+
"<extra_id_59>",
|
63 |
+
"<extra_id_60>",
|
64 |
+
"<extra_id_61>",
|
65 |
+
"<extra_id_62>",
|
66 |
+
"<extra_id_63>",
|
67 |
+
"<extra_id_64>",
|
68 |
+
"<extra_id_65>",
|
69 |
+
"<extra_id_66>",
|
70 |
+
"<extra_id_67>",
|
71 |
+
"<extra_id_68>",
|
72 |
+
"<extra_id_69>",
|
73 |
+
"<extra_id_70>",
|
74 |
+
"<extra_id_71>",
|
75 |
+
"<extra_id_72>",
|
76 |
+
"<extra_id_73>",
|
77 |
+
"<extra_id_74>",
|
78 |
+
"<extra_id_75>",
|
79 |
+
"<extra_id_76>",
|
80 |
+
"<extra_id_77>",
|
81 |
+
"<extra_id_78>",
|
82 |
+
"<extra_id_79>",
|
83 |
+
"<extra_id_80>",
|
84 |
+
"<extra_id_81>",
|
85 |
+
"<extra_id_82>",
|
86 |
+
"<extra_id_83>",
|
87 |
+
"<extra_id_84>",
|
88 |
+
"<extra_id_85>",
|
89 |
+
"<extra_id_86>",
|
90 |
+
"<extra_id_87>",
|
91 |
+
"<extra_id_88>",
|
92 |
+
"<extra_id_89>",
|
93 |
+
"<extra_id_90>",
|
94 |
+
"<extra_id_91>",
|
95 |
+
"<extra_id_92>",
|
96 |
+
"<extra_id_93>",
|
97 |
+
"<extra_id_94>",
|
98 |
+
"<extra_id_95>",
|
99 |
+
"<extra_id_96>",
|
100 |
+
"<extra_id_97>",
|
101 |
+
"<extra_id_98>",
|
102 |
+
"<extra_id_99>"
|
103 |
+
],
|
104 |
+
"clean_up_tokenization_spaces": true,
|
105 |
+
"eos_token": "</s>",
|
106 |
+
"extra_ids": 100,
|
107 |
+
"legacy": false,
|
108 |
+
"model_max_length": 1000000000000000019884624838656,
|
109 |
+
"pad_token": "<pad>",
|
110 |
+
"tokenizer_class": "T5Tokenizer",
|
111 |
+
"unk_token": "<unk>"
|
112 |
+
}
|