Upload tokenizer

added_tokens.json

@@ -0,0 +1,302 @@
+{
+  "<extra_id_0>": 256299,
+  "<extra_id_100>": 256199,
+  "<extra_id_101>": 256198,
+  "<extra_id_102>": 256197,
+  "<extra_id_103>": 256196,
+  "<extra_id_104>": 256195,
+  "<extra_id_105>": 256194,
+  "<extra_id_106>": 256193,
+  "<extra_id_107>": 256192,
+  "<extra_id_108>": 256191,
+  "<extra_id_109>": 256190,
+  "<extra_id_10>": 256289,
+  "<extra_id_110>": 256189,
+  "<extra_id_111>": 256188,
+  "<extra_id_112>": 256187,
+  "<extra_id_113>": 256186,
+  "<extra_id_114>": 256185,
+  "<extra_id_115>": 256184,
+  "<extra_id_116>": 256183,
+  "<extra_id_117>": 256182,
+  "<extra_id_118>": 256181,
+  "<extra_id_119>": 256180,
+  "<extra_id_11>": 256288,
+  "<extra_id_120>": 256179,
+  "<extra_id_121>": 256178,
+  "<extra_id_122>": 256177,
+  "<extra_id_123>": 256176,
+  "<extra_id_124>": 256175,
+  "<extra_id_125>": 256174,
+  "<extra_id_126>": 256173,
+  "<extra_id_127>": 256172,
+  "<extra_id_128>": 256171,
+  "<extra_id_129>": 256170,
+  "<extra_id_12>": 256287,
+  "<extra_id_130>": 256169,
+  "<extra_id_131>": 256168,
+  "<extra_id_132>": 256167,
+  "<extra_id_133>": 256166,
+  "<extra_id_134>": 256165,
+  "<extra_id_135>": 256164,
+  "<extra_id_136>": 256163,
+  "<extra_id_137>": 256162,
+  "<extra_id_138>": 256161,
+  "<extra_id_139>": 256160,
+  "<extra_id_13>": 256286,
+  "<extra_id_140>": 256159,
+  "<extra_id_141>": 256158,
+  "<extra_id_142>": 256157,
+  "<extra_id_143>": 256156,
+  "<extra_id_144>": 256155,
+  "<extra_id_145>": 256154,
+  "<extra_id_146>": 256153,
+  "<extra_id_147>": 256152,
+  "<extra_id_148>": 256151,
+  "<extra_id_149>": 256150,
+  "<extra_id_14>": 256285,
+  "<extra_id_150>": 256149,
+  "<extra_id_151>": 256148,
+  "<extra_id_152>": 256147,
+  "<extra_id_153>": 256146,
+  "<extra_id_154>": 256145,
+  "<extra_id_155>": 256144,
+  "<extra_id_156>": 256143,
+  "<extra_id_157>": 256142,
+  "<extra_id_158>": 256141,
+  "<extra_id_159>": 256140,
+  "<extra_id_15>": 256284,
+  "<extra_id_160>": 256139,
+  "<extra_id_161>": 256138,
+  "<extra_id_162>": 256137,
+  "<extra_id_163>": 256136,
+  "<extra_id_164>": 256135,
+  "<extra_id_165>": 256134,
+  "<extra_id_166>": 256133,
+  "<extra_id_167>": 256132,
+  "<extra_id_168>": 256131,
+  "<extra_id_169>": 256130,
+  "<extra_id_16>": 256283,
+  "<extra_id_170>": 256129,
+  "<extra_id_171>": 256128,
+  "<extra_id_172>": 256127,
+  "<extra_id_173>": 256126,
+  "<extra_id_174>": 256125,
+  "<extra_id_175>": 256124,
+  "<extra_id_176>": 256123,
+  "<extra_id_177>": 256122,
+  "<extra_id_178>": 256121,
+  "<extra_id_179>": 256120,
+  "<extra_id_17>": 256282,
+  "<extra_id_180>": 256119,
+  "<extra_id_181>": 256118,
+  "<extra_id_182>": 256117,
+  "<extra_id_183>": 256116,
+  "<extra_id_184>": 256115,
+  "<extra_id_185>": 256114,
+  "<extra_id_186>": 256113,
+  "<extra_id_187>": 256112,
+  "<extra_id_188>": 256111,
+  "<extra_id_189>": 256110,
+  "<extra_id_18>": 256281,
+  "<extra_id_190>": 256109,
+  "<extra_id_191>": 256108,
+  "<extra_id_192>": 256107,
+  "<extra_id_193>": 256106,
+  "<extra_id_194>": 256105,
+  "<extra_id_195>": 256104,
+  "<extra_id_196>": 256103,
+  "<extra_id_197>": 256102,
+  "<extra_id_198>": 256101,
+  "<extra_id_199>": 256100,
+  "<extra_id_19>": 256280,
+  "<extra_id_1>": 256298,
+  "<extra_id_200>": 256099,
+  "<extra_id_201>": 256098,
+  "<extra_id_202>": 256097,
+  "<extra_id_203>": 256096,
+  "<extra_id_204>": 256095,
+  "<extra_id_205>": 256094,
+  "<extra_id_206>": 256093,
+  "<extra_id_207>": 256092,
+  "<extra_id_208>": 256091,
+  "<extra_id_209>": 256090,
+  "<extra_id_20>": 256279,
+  "<extra_id_210>": 256089,
+  "<extra_id_211>": 256088,
+  "<extra_id_212>": 256087,
+  "<extra_id_213>": 256086,
+  "<extra_id_214>": 256085,
+  "<extra_id_215>": 256084,
+  "<extra_id_216>": 256083,
+  "<extra_id_217>": 256082,
+  "<extra_id_218>": 256081,
+  "<extra_id_219>": 256080,
+  "<extra_id_21>": 256278,
+  "<extra_id_220>": 256079,
+  "<extra_id_221>": 256078,
+  "<extra_id_222>": 256077,
+  "<extra_id_223>": 256076,
+  "<extra_id_224>": 256075,
+  "<extra_id_225>": 256074,
+  "<extra_id_226>": 256073,
+  "<extra_id_227>": 256072,
+  "<extra_id_228>": 256071,
+  "<extra_id_229>": 256070,
+  "<extra_id_22>": 256277,
+  "<extra_id_230>": 256069,
+  "<extra_id_231>": 256068,
+  "<extra_id_232>": 256067,
+  "<extra_id_233>": 256066,
+  "<extra_id_234>": 256065,
+  "<extra_id_235>": 256064,
+  "<extra_id_236>": 256063,
+  "<extra_id_237>": 256062,
+  "<extra_id_238>": 256061,
+  "<extra_id_239>": 256060,
+  "<extra_id_23>": 256276,
+  "<extra_id_240>": 256059,
+  "<extra_id_241>": 256058,
+  "<extra_id_242>": 256057,
+  "<extra_id_243>": 256056,
+  "<extra_id_244>": 256055,
+  "<extra_id_245>": 256054,
+  "<extra_id_246>": 256053,
+  "<extra_id_247>": 256052,
+  "<extra_id_248>": 256051,
+  "<extra_id_249>": 256050,
+  "<extra_id_24>": 256275,
+  "<extra_id_250>": 256049,
+  "<extra_id_251>": 256048,
+  "<extra_id_252>": 256047,
+  "<extra_id_253>": 256046,
+  "<extra_id_254>": 256045,
+  "<extra_id_255>": 256044,
+  "<extra_id_256>": 256043,
+  "<extra_id_257>": 256042,
+  "<extra_id_258>": 256041,
+  "<extra_id_259>": 256040,
+  "<extra_id_25>": 256274,
+  "<extra_id_260>": 256039,
+  "<extra_id_261>": 256038,
+  "<extra_id_262>": 256037,
+  "<extra_id_263>": 256036,
+  "<extra_id_264>": 256035,
+  "<extra_id_265>": 256034,
+  "<extra_id_266>": 256033,
+  "<extra_id_267>": 256032,
+  "<extra_id_268>": 256031,
+  "<extra_id_269>": 256030,
+  "<extra_id_26>": 256273,
+  "<extra_id_270>": 256029,
+  "<extra_id_271>": 256028,
+  "<extra_id_272>": 256027,
+  "<extra_id_273>": 256026,
+  "<extra_id_274>": 256025,
+  "<extra_id_275>": 256024,
+  "<extra_id_276>": 256023,
+  "<extra_id_277>": 256022,
+  "<extra_id_278>": 256021,
+  "<extra_id_279>": 256020,
+  "<extra_id_27>": 256272,
+  "<extra_id_280>": 256019,
+  "<extra_id_281>": 256018,
+  "<extra_id_282>": 256017,
+  "<extra_id_283>": 256016,
+  "<extra_id_284>": 256015,
+  "<extra_id_285>": 256014,
+  "<extra_id_286>": 256013,
+  "<extra_id_287>": 256012,
+  "<extra_id_288>": 256011,
+  "<extra_id_289>": 256010,
+  "<extra_id_28>": 256271,
+  "<extra_id_290>": 256009,
+  "<extra_id_291>": 256008,
+  "<extra_id_292>": 256007,
+  "<extra_id_293>": 256006,
+  "<extra_id_294>": 256005,
+  "<extra_id_295>": 256004,
+  "<extra_id_296>": 256003,
+  "<extra_id_297>": 256002,
+  "<extra_id_298>": 256001,
+  "<extra_id_299>": 256000,
+  "<extra_id_29>": 256270,
+  "<extra_id_2>": 256297,
+  "<extra_id_30>": 256269,
+  "<extra_id_31>": 256268,
+  "<extra_id_32>": 256267,
+  "<extra_id_33>": 256266,
+  "<extra_id_34>": 256265,
+  "<extra_id_35>": 256264,
+  "<extra_id_36>": 256263,
+  "<extra_id_37>": 256262,
+  "<extra_id_38>": 256261,
+  "<extra_id_39>": 256260,
+  "<extra_id_3>": 256296,
+  "<extra_id_40>": 256259,
+  "<extra_id_41>": 256258,
+  "<extra_id_42>": 256257,
+  "<extra_id_43>": 256256,
+  "<extra_id_44>": 256255,
+  "<extra_id_45>": 256254,
+  "<extra_id_46>": 256253,
+  "<extra_id_47>": 256252,
+  "<extra_id_48>": 256251,
+  "<extra_id_49>": 256250,
+  "<extra_id_4>": 256295,
+  "<extra_id_50>": 256249,
+  "<extra_id_51>": 256248,
+  "<extra_id_52>": 256247,
+  "<extra_id_53>": 256246,
+  "<extra_id_54>": 256245,
+  "<extra_id_55>": 256244,
+  "<extra_id_56>": 256243,
+  "<extra_id_57>": 256242,
+  "<extra_id_58>": 256241,
+  "<extra_id_59>": 256240,
+  "<extra_id_5>": 256294,
+  "<extra_id_60>": 256239,
+  "<extra_id_61>": 256238,
+  "<extra_id_62>": 256237,
+  "<extra_id_63>": 256236,
+  "<extra_id_64>": 256235,
+  "<extra_id_65>": 256234,
+  "<extra_id_66>": 256233,
+  "<extra_id_67>": 256232,
+  "<extra_id_68>": 256231,
+  "<extra_id_69>": 256230,
+  "<extra_id_6>": 256293,
+  "<extra_id_70>": 256229,
+  "<extra_id_71>": 256228,
+  "<extra_id_72>": 256227,
+  "<extra_id_73>": 256226,
+  "<extra_id_74>": 256225,
+  "<extra_id_75>": 256224,
+  "<extra_id_76>": 256223,
+  "<extra_id_77>": 256222,
+  "<extra_id_78>": 256221,
+  "<extra_id_79>": 256220,
+  "<extra_id_7>": 256292,
+  "<extra_id_80>": 256219,
+  "<extra_id_81>": 256218,
+  "<extra_id_82>": 256217,
+  "<extra_id_83>": 256216,
+  "<extra_id_84>": 256215,
+  "<extra_id_85>": 256214,
+  "<extra_id_86>": 256213,
+  "<extra_id_87>": 256212,
+  "<extra_id_88>": 256211,
+  "<extra_id_89>": 256210,
+  "<extra_id_8>": 256291,
+  "<extra_id_90>": 256209,
+  "<extra_id_91>": 256208,
+  "<extra_id_92>": 256207,
+  "<extra_id_93>": 256206,
+  "<extra_id_94>": 256205,
+  "<extra_id_95>": 256204,
+  "<extra_id_96>": 256203,
+  "<extra_id_97>": 256202,
+  "<extra_id_98>": 256201,
+  "<extra_id_99>": 256200,
+  "<extra_id_9>": 256290
+}

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3909a67b780650b35cf529ac782ad2b6b26e6d1f849d3fbb6a872905f452458
+size 4548313

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": null,
+  "sp_model_kwargs": {},
+  "tokenizer_class": "BertGenerationTokenizer",
+  "unk_token": "<unk>"
+}