



import unittest

from transformers import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING, AutoTokenizer, is_vision_available
from transformers.pipelines import pipeline
from transformers.pipelines.document_question_answering import apply_tesseract
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_detectrona,
    require_pytesseract,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
  from PIL import Image

  from transformers.image_utils import load_image
else:



  class  __lowerCAmelCase     :




   @staticmethod
   def        lowerCamelCase      (						*__UpperCAmelCase	,				**__UpperCAmelCase				):



      '''simple docstring'''
      pass





  def 		a__			(		_UpperCamelCase			: List[str]							):
     return None


# This is a pinned image from a specific revision of a document question answering space, hosted by HuggingFace,
# so we can expect it to be available.
a_			=					(
    """https://huggingface.co/spaces/impira/docquery/resolve/2f6c96314dc84dfda62d40de9da55f2f5165d403/invoice.png"""
)



@is_pipeline_test
@require_torch
@require_vision
class  __lowerCAmelCase     (	unittest.TestCase   ):
 lowerCAmelCase__       					=  MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING




 @require_pytesseract
 @require_vision
 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    pipeline(
        '''document-question-answering'''	,				model=__UpperCAmelCase	,				tokenizer=__UpperCAmelCase	,				image_processor=__UpperCAmelCase				)

    __lowerCamelCase													=    INVOICE_URL
    __lowerCamelCase													=    list(zip(*apply_tesseract(load_image(__UpperCAmelCase				)	,				__UpperCAmelCase	,				''''''				)				)				)
    __lowerCamelCase													=    '''What is the placebo?'''
    __lowerCamelCase													=    [
        {
            '''image''': load_image(__UpperCAmelCase				),
            '''question''': question,
        },
        {
            '''image''': image,
            '''question''': question,
        },
        {
            '''image''': image,
            '''question''': question,
            '''word_boxes''': word_boxes,
        },
    ]
    return dqa_pipeline, examples




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    dqa_pipeline(__UpperCAmelCase	,				top_k=2				)
    self.assertEqual(
        __UpperCAmelCase	,				[
            [
                {'''score''': ANY(__UpperCAmelCase				), '''answer''': ANY(__UpperCAmelCase				), '''start''': ANY(__UpperCAmelCase				), '''end''': ANY(__UpperCAmelCase				)},
                {'''score''': ANY(__UpperCAmelCase				), '''answer''': ANY(__UpperCAmelCase				), '''start''': ANY(__UpperCAmelCase				), '''end''': ANY(__UpperCAmelCase				)},
            ]
        ]
        * 3	,				)




 @require_torch
 @require_detectrona
 @require_pytesseract
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    pipeline('''document-question-answering'''	,				model='''hf-internal-testing/tiny-random-layoutlmv2'''				)
    __lowerCamelCase													=    INVOICE_URL
    __lowerCamelCase													=    '''How many cats are there?'''

    __lowerCamelCase													=    [
        {'''score''': 0.0_001, '''answer''': '''oy 2312/2019''', '''start''': 38, '''end''': 39},
        {'''score''': 0.0_001, '''answer''': '''oy 2312/2019 DUE''', '''start''': 38, '''end''': 40},
    ]
    __lowerCamelCase													=    dqa_pipeline(image=__UpperCAmelCase	,				question=__UpperCAmelCase	,				top_k=2				)
    self.assertEqual(nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				__UpperCAmelCase				)

    __lowerCamelCase													=    dqa_pipeline({'''image''': image, '''question''': question}	,				top_k=2				)
    self.assertEqual(nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				__UpperCAmelCase				)

    # This image does not detect ANY text in it, meaning layoutlmv2 should fail.
    # Empty answer probably
    __lowerCamelCase													=    '''./tests/fixtures/tests_samples/COCO/000000039769.png'''
    __lowerCamelCase													=    dqa_pipeline(image=__UpperCAmelCase	,				question=__UpperCAmelCase	,				top_k=2				)
    self.assertEqual(__UpperCAmelCase	,				[]				)

    # We can optionnally pass directly the words and bounding boxes
    __lowerCamelCase													=    '''./tests/fixtures/tests_samples/COCO/000000039769.png'''
    __lowerCamelCase													=    []
    __lowerCamelCase													=    []
    __lowerCamelCase													=    dqa_pipeline(image=__UpperCAmelCase	,				question=__UpperCAmelCase	,				words=__UpperCAmelCase	,				boxes=__UpperCAmelCase	,				top_k=2				)
    self.assertEqual(__UpperCAmelCase	,				[]				)




 @slow
 @require_torch
 @require_detectrona
 @require_pytesseract
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    pipeline(
        '''document-question-answering'''	,				model='''tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa'''	,				revision='''9977165'''	,				)
    __lowerCamelCase													=    INVOICE_URL
    __lowerCamelCase													=    '''What is the invoice number?'''

    __lowerCamelCase													=    dqa_pipeline(image=__UpperCAmelCase	,				question=__UpperCAmelCase	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            {'''score''': 0.9_944, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
            {'''score''': 0.0_009, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
        ]	,				)

    __lowerCamelCase													=    dqa_pipeline({'''image''': image, '''question''': question}	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            {'''score''': 0.9_944, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
            {'''score''': 0.0_009, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
        ]	,				)

    __lowerCamelCase													=    dqa_pipeline(
        [{'''image''': image, '''question''': question}, {'''image''': image, '''question''': question}]	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            [
                {'''score''': 0.9_944, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
                {'''score''': 0.0_009, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
            ],
        ]
        * 2	,				)




 @slow
 @require_torch
 @require_detectrona
 @require_pytesseract
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    pipeline(
        '''document-question-answering'''	,				model='''tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa'''	,				revision='''9977165'''	,				max_seq_len=50	,				)
    __lowerCamelCase													=    INVOICE_URL
    __lowerCamelCase													=    '''What is the invoice number?'''

    __lowerCamelCase													=    dqa_pipeline(image=__UpperCAmelCase	,				question=__UpperCAmelCase	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            {'''score''': 0.9_974, '''answer''': '''1110212019''', '''start''': 23, '''end''': 23},
            {'''score''': 0.9_948, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
        ]	,				)

    __lowerCamelCase													=    dqa_pipeline({'''image''': image, '''question''': question}	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            {'''score''': 0.9_974, '''answer''': '''1110212019''', '''start''': 23, '''end''': 23},
            {'''score''': 0.9_948, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
        ]	,				)

    __lowerCamelCase													=    dqa_pipeline(
        [{'''image''': image, '''question''': question}, {'''image''': image, '''question''': question}]	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            [
                {'''score''': 0.9_974, '''answer''': '''1110212019''', '''start''': 23, '''end''': 23},
                {'''score''': 0.9_948, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
            ]
        ]
        * 2	,				)




 @slow
 @require_torch
 @require_pytesseract
 @require_vision
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    AutoTokenizer.from_pretrained(
        '''impira/layoutlm-document-qa'''	,				revision='''3dc6de3'''	,				add_prefix_space=__UpperCAmelCase				)
    __lowerCamelCase													=    pipeline(
        '''document-question-answering'''	,				model='''impira/layoutlm-document-qa'''	,				tokenizer=__UpperCAmelCase	,				revision='''3dc6de3'''	,				)
    __lowerCamelCase													=    INVOICE_URL
    __lowerCamelCase													=    '''What is the invoice number?'''

    __lowerCamelCase													=    dqa_pipeline(image=__UpperCAmelCase	,				question=__UpperCAmelCase	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            {'''score''': 0.4_251, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
            {'''score''': 0.0_819, '''answer''': '''1110212019''', '''start''': 23, '''end''': 23},
        ]	,				)

    __lowerCamelCase													=    dqa_pipeline({'''image''': image, '''question''': question}	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            {'''score''': 0.4_251, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
            {'''score''': 0.0_819, '''answer''': '''1110212019''', '''start''': 23, '''end''': 23},
        ]	,				)

    __lowerCamelCase													=    dqa_pipeline(
        [{'''image''': image, '''question''': question}, {'''image''': image, '''question''': question}]	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            [
                {'''score''': 0.4_251, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
                {'''score''': 0.0_819, '''answer''': '''1110212019''', '''start''': 23, '''end''': 23},
            ]
        ]
        * 2	,				)

    __lowerCamelCase													=    list(zip(*apply_tesseract(load_image(__UpperCAmelCase				)	,				__UpperCAmelCase	,				''''''				)				)				)

    # This model should also work if `image` is set to None
    __lowerCamelCase													=    dqa_pipeline({'''image''': None, '''word_boxes''': word_boxes, '''question''': question}	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            {'''score''': 0.4_251, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
            {'''score''': 0.0_819, '''answer''': '''1110212019''', '''start''': 23, '''end''': 23},
        ]	,				)




 @slow
 @require_torch
 @require_pytesseract
 @require_vision
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    AutoTokenizer.from_pretrained(
        '''impira/layoutlm-document-qa'''	,				revision='''3dc6de3'''	,				add_prefix_space=__UpperCAmelCase				)
    __lowerCamelCase													=    pipeline(
        '''document-question-answering'''	,				model='''impira/layoutlm-document-qa'''	,				tokenizer=__UpperCAmelCase	,				revision='''3dc6de3'''	,				max_seq_len=50	,				)
    __lowerCamelCase													=    INVOICE_URL
    __lowerCamelCase													=    '''What is the invoice number?'''

    __lowerCamelCase													=    dqa_pipeline(image=__UpperCAmelCase	,				question=__UpperCAmelCase	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            {'''score''': 0.9_999, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
            {'''score''': 0.9_998, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
        ]	,				)

    __lowerCamelCase													=    dqa_pipeline(
        [{'''image''': image, '''question''': question}, {'''image''': image, '''question''': question}]	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            [
                {'''score''': 0.9_999, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
                {'''score''': 0.9_998, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
            ]
        ]
        * 2	,				)

    __lowerCamelCase													=    list(zip(*apply_tesseract(load_image(__UpperCAmelCase				)	,				__UpperCAmelCase	,				''''''				)				)				)

    # This model should also work if `image` is set to None
    __lowerCamelCase													=    dqa_pipeline({'''image''': None, '''word_boxes''': word_boxes, '''question''': question}	,				top_k=2				)
    self.assertEqual(
        nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[
            {'''score''': 0.9_999, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
            {'''score''': 0.9_998, '''answer''': '''us-001''', '''start''': 16, '''end''': 16},
        ]	,				)




 @slow
 @require_torch
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    pipeline(
        '''document-question-answering'''	,				model='''naver-clova-ix/donut-base-finetuned-docvqa'''	,				tokenizer=AutoTokenizer.from_pretrained('''naver-clova-ix/donut-base-finetuned-docvqa'''				)	,				feature_extractor='''naver-clova-ix/donut-base-finetuned-docvqa'''	,				)

    __lowerCamelCase													=    INVOICE_URL
    __lowerCamelCase													=    '''What is the invoice number?'''
    __lowerCamelCase													=    dqa_pipeline(image=__UpperCAmelCase	,				question=__UpperCAmelCase	,				top_k=2				)
    self.assertEqual(nested_simplify(__UpperCAmelCase	,				decimals=4				)	,				[{'''answer''': '''us-001'''}]				)




 @require_tf
 @unittest.skip('''Document question answering not implemented in TF'''				)
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    pass











import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a_			=					get_tests_dir("""fixtures/test_sentencepiece.model""")



@require_sentencepiece
class  __lowerCAmelCase     (	lowerCAmelCase__  ,	unittest.TestCase   ):
 lowerCAmelCase__       					=  XLMProphetNetTokenizer
 lowerCAmelCase__       					=  False
 lowerCAmelCase__       					=  True




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    super().setUp()

    # We have a SentencePiece fixture for testing
    __lowerCamelCase													=    XLMProphetNetTokenizer(__UpperCAmelCase	,				keep_accents=__UpperCAmelCase				)
    tokenizer.save_pretrained(self.tmpdirname				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''[PAD]'''
    __lowerCamelCase													=    0

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCAmelCase				)	,				__UpperCAmelCase				)
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCAmelCase				)	,				__UpperCAmelCase				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    list(self.get_tokenizer().get_vocab().keys()				)

    self.assertEqual(vocab_keys[0]	,				'''[PAD]'''				)
    self.assertEqual(vocab_keys[1]	,				'''[CLS]'''				)
    self.assertEqual(vocab_keys[-1]	,				'''j'''				)
    self.assertEqual(len(__UpperCAmelCase				)	,				1012				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    self.assertEqual(self.get_tokenizer().vocab_size	,				1012				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    XLMProphetNetTokenizer(__UpperCAmelCase	,				keep_accents=__UpperCAmelCase				)

    __lowerCamelCase													=    tokenizer.tokenize('''This is a test'''				)
    self.assertListEqual(__UpperCAmelCase	,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)	,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,				)

    __lowerCamelCase													=    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''9''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''é''',
            '''.''',
        ]	,				)
    __lowerCamelCase													=    tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            value + tokenizer.fairseq_offset
            for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
        ]	,				)

    __lowerCamelCase													=    tokenizer.convert_ids_to_tokens(__UpperCAmelCase				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''[UNK]''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''[UNK]''',
            '''.''',
        ]	,				)




 @cached_property
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''Hello World!'''
    __lowerCamelCase													=    [35389, 6672, 49, 2]
    self.assertListEqual(__UpperCAmelCase	,				self.big_tokenizer.encode(__UpperCAmelCase				)				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    # fmt: off
    __lowerCamelCase													=    {'''input_ids''': [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
    # fmt: on

    self.tokenizer_integration_test_util(
        expected_encoding=__UpperCAmelCase	,				model_name='''microsoft/xprophetnet-large-wiki100-cased'''	,				revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''	,				)






