



import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a_			=					get_tests_dir("""fixtures/test_sentencepiece.model""")



@require_sentencepiece
@require_tokenizers
class  __lowerCAmelCase     (	__SCREAMING_SNAKE_CASE  ,	unittest.TestCase   ):
 lowerCAmelCase__       					=  XLMRobertaTokenizer
 lowerCAmelCase__       					=  XLMRobertaTokenizerFast
 lowerCAmelCase__       					=  True
 lowerCAmelCase__       					=  True




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    super().setUp()

    # We have a SentencePiece fixture for testing
    __lowerCamelCase													=    XLMRobertaTokenizer(__snake_case	,				keep_accents=__snake_case				)
    tokenizer.save_pretrained(self.tmpdirname				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''<pad>'''
    __lowerCamelCase													=    1

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case				)	,				__snake_case				)
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case				)	,				__snake_case				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    list(self.get_tokenizer().get_vocab().keys()				)

    self.assertEqual(vocab_keys[0]	,				'''<s>'''				)
    self.assertEqual(vocab_keys[1]	,				'''<pad>'''				)
    self.assertEqual(vocab_keys[-1]	,				'''<mask>'''				)
    self.assertEqual(len(__snake_case				)	,				1002				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    self.assertEqual(self.get_tokenizer().vocab_size	,				1002				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    XLMRobertaTokenizer(__snake_case	,				keep_accents=__snake_case				)

    __lowerCamelCase													=    tokenizer.tokenize('''This is a test'''				)
    self.assertListEqual(__snake_case	,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(__snake_case				)	,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,				)

    __lowerCamelCase													=    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
    self.assertListEqual(
        __snake_case	,				[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''9''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''é''',
            '''.''',
        ]	,				)
    __lowerCamelCase													=    tokenizer.convert_tokens_to_ids(__snake_case				)
    self.assertListEqual(
        __snake_case	,				[
            value + tokenizer.fairseq_offset
            for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
        ]	,				)

    __lowerCamelCase													=    tokenizer.convert_ids_to_tokens(__snake_case				)
    self.assertListEqual(
        __snake_case	,				[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''<unk>''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''<unk>''',
            '''.''',
        ]	,				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    if not self.test_slow_tokenizer:
       # as we don't have a slow version, we can't compare the outputs between slow and fast versions
       return

    __lowerCamelCase													=    (self.rust_tokenizer_class, '''hf-internal-testing/tiny-xlm-roberta''', {})
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
          __lowerCamelCase													=    self.rust_tokenizer_class.from_pretrained(__snake_case	,				**__snake_case				)
          __lowerCamelCase													=    self.tokenizer_class.from_pretrained(__snake_case	,				**__snake_case				)

          __lowerCamelCase													=    tempfile.mkdtemp()

          __lowerCamelCase													=    tokenizer_r.save_pretrained(__snake_case				)
          __lowerCamelCase													=    tokenizer_p.save_pretrained(__snake_case				)

          # Checks it save with the same files + the tokenizer.json file for the fast one
          self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files				)				)
          __lowerCamelCase													=    tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f				)
          self.assertSequenceEqual(__snake_case	,				__snake_case				)

          # Checks everything loads correctly in the same way
          __lowerCamelCase													=    tokenizer_r.from_pretrained(__snake_case				)
          __lowerCamelCase													=    tokenizer_p.from_pretrained(__snake_case				)

          # Check special tokens are set accordingly on Rust and Python
          for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(__snake_case	,				__snake_case				)				)
             # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
             # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

          shutil.rmtree(__snake_case				)

          # Save tokenizer rust, legacy_format=True
          __lowerCamelCase													=    tempfile.mkdtemp()

          __lowerCamelCase													=    tokenizer_r.save_pretrained(__snake_case	,				legacy_format=__snake_case				)
          __lowerCamelCase													=    tokenizer_p.save_pretrained(__snake_case				)

          # Checks it save with the same files
          self.assertSequenceEqual(__snake_case	,				__snake_case				)

          # Checks everything loads correctly in the same way
          __lowerCamelCase													=    tokenizer_r.from_pretrained(__snake_case				)
          __lowerCamelCase													=    tokenizer_p.from_pretrained(__snake_case				)

          # Check special tokens are set accordingly on Rust and Python
          for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(__snake_case	,				__snake_case				)				)

          shutil.rmtree(__snake_case				)

          # Save tokenizer rust, legacy_format=False
          __lowerCamelCase													=    tempfile.mkdtemp()

          __lowerCamelCase													=    tokenizer_r.save_pretrained(__snake_case	,				legacy_format=__snake_case				)
          __lowerCamelCase													=    tokenizer_p.save_pretrained(__snake_case				)

          # Checks it saved the tokenizer.json file
          self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files				)				)

          # Checks everything loads correctly in the same way
          __lowerCamelCase													=    tokenizer_r.from_pretrained(__snake_case				)
          __lowerCamelCase													=    tokenizer_p.from_pretrained(__snake_case				)

          # Check special tokens are set accordingly on Rust and Python
          for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(__snake_case	,				__snake_case				)				)

          shutil.rmtree(__snake_case				)




 @cached_property
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return XLMRobertaTokenizer.from_pretrained('''xlm-roberta-base'''				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    with tempfile.NamedTemporaryFile() as f:
       shutil.copyfile(__snake_case	,				f.name				)
       __lowerCamelCase													=    XLMRobertaTokenizer(f.name	,				keep_accents=__snake_case				)
       __lowerCamelCase													=    pickle.dumps(__snake_case				)
    pickle.loads(__snake_case				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    if not self.test_rust_tokenizer:
       return

    __lowerCamelCase													=    self.get_tokenizer()
    __lowerCamelCase													=    self.get_rust_tokenizer()

    __lowerCamelCase													=    '''I was born in 92000, and this is falsé.'''

    __lowerCamelCase													=    tokenizer.tokenize(__snake_case				)
    __lowerCamelCase													=    rust_tokenizer.tokenize(__snake_case				)
    self.assertListEqual(__snake_case	,				__snake_case				)

    __lowerCamelCase													=    tokenizer.encode(__snake_case	,				add_special_tokens=__snake_case				)
    __lowerCamelCase													=    rust_tokenizer.encode(__snake_case	,				add_special_tokens=__snake_case				)
    self.assertListEqual(__snake_case	,				__snake_case				)

    __lowerCamelCase													=    self.get_rust_tokenizer()
    __lowerCamelCase													=    tokenizer.encode(__snake_case				)
    __lowerCamelCase													=    rust_tokenizer.encode(__snake_case				)
    self.assertListEqual(__snake_case	,				__snake_case				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''Hello World!'''
    __lowerCamelCase													=    [0, 35378, 6661, 38, 2]
    # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
    # xlmr.eval()
    # xlmr.encode(symbols)

    self.assertListEqual(__snake_case	,				self.big_tokenizer.encode(__snake_case				)				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    (
        '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
        ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
    )
    __lowerCamelCase													=    [
        0,
        3293,
        83,
        10,
        4552,
        4989,
        7986,
        678,
        10,
        5915,
        111,
        179459,
        124850,
        4,
        6044,
        237,
        12,
        6,
        5,
        6,
        4,
        6780,
        705,
        15,
        1388,
        44,
        378,
        10114,
        711,
        152,
        20,
        6,
        5,
        22376,
        642,
        1221,
        15190,
        34153,
        450,
        5608,
        959,
        1119,
        57702,
        136,
        186,
        47,
        1098,
        29367,
        47,
        # 4426, # What fairseq tokenizes from "<unk>": "_<"
        # 3678, # What fairseq tokenizes from "<unk>": "unk"
        # 2740, # What fairseq tokenizes from "<unk>": ">"
        3,  # What we tokenize from "<unk>": "<unk>"
        6,  # Residue from the tokenization: an extra sentencepiece underline
        4,
        6044,
        237,
        6284,
        50901,
        528,
        31,
        90,
        34,
        927,
        2,
    ]
    # xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
    # xlmr.eval()
    # xlmr.encode(symbols)

    self.assertListEqual(__snake_case	,				self.big_tokenizer.encode(__snake_case				)				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    # fmt: off
    __lowerCamelCase													=    {'''input_ids''': [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
    # fmt: on

    self.tokenizer_integration_test_util(
        expected_encoding=__snake_case	,				model_name='''xlm-roberta-base'''	,				revision='''d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3'''	,				)











import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a_			=					get_tests_dir("""fixtures/test_sentencepiece.model""")



@require_sentencepiece
class  __lowerCAmelCase     (	lowerCAmelCase__  ,	unittest.TestCase   ):
 lowerCAmelCase__       					=  XLMProphetNetTokenizer
 lowerCAmelCase__       					=  False
 lowerCAmelCase__       					=  True




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    super().setUp()

    # We have a SentencePiece fixture for testing
    __lowerCamelCase													=    XLMProphetNetTokenizer(__UpperCAmelCase	,				keep_accents=__UpperCAmelCase				)
    tokenizer.save_pretrained(self.tmpdirname				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''[PAD]'''
    __lowerCamelCase													=    0

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCAmelCase				)	,				__UpperCAmelCase				)
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCAmelCase				)	,				__UpperCAmelCase				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    list(self.get_tokenizer().get_vocab().keys()				)

    self.assertEqual(vocab_keys[0]	,				'''[PAD]'''				)
    self.assertEqual(vocab_keys[1]	,				'''[CLS]'''				)
    self.assertEqual(vocab_keys[-1]	,				'''j'''				)
    self.assertEqual(len(__UpperCAmelCase				)	,				1012				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    self.assertEqual(self.get_tokenizer().vocab_size	,				1012				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    XLMProphetNetTokenizer(__UpperCAmelCase	,				keep_accents=__UpperCAmelCase				)

    __lowerCamelCase													=    tokenizer.tokenize('''This is a test'''				)
    self.assertListEqual(__UpperCAmelCase	,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)	,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,				)

    __lowerCamelCase													=    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''9''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''é''',
            '''.''',
        ]	,				)
    __lowerCamelCase													=    tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            value + tokenizer.fairseq_offset
            for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
        ]	,				)

    __lowerCamelCase													=    tokenizer.convert_ids_to_tokens(__UpperCAmelCase				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''[UNK]''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''[UNK]''',
            '''.''',
        ]	,				)




 @cached_property
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''Hello World!'''
    __lowerCamelCase													=    [35389, 6672, 49, 2]
    self.assertListEqual(__UpperCAmelCase	,				self.big_tokenizer.encode(__UpperCAmelCase				)				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    # fmt: off
    __lowerCamelCase													=    {'''input_ids''': [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
    # fmt: on

    self.tokenizer_integration_test_util(
        expected_encoding=__UpperCAmelCase	,				model_name='''microsoft/xprophetnet-large-wiki100-cased'''	,				revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''	,				)






