





import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 			__A       (      lowerCamelCase__ ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      LxmertTokenizer
      UpperCAmelCase__						=      LxmertTokenizerFast
      UpperCAmelCase__						=      True
      UpperCAmelCase__						=      True


      def 				__snake_case			(		self):



             """simple docstring"""





             super().setUp()

             _lowerCamelCase							:     List[Any]	=		[
                 '''[UNK]''',
                 '''[CLS]''',
                 '''[SEP]''',
                 '''want''',
                 '''##want''',
                 '''##ed''',
                 '''wa''',
                 '''un''',
                 '''runn''',
                 '''##ing''',
                 ''',''',
                 '''low''',
                 '''lowest''',
             ]
             _lowerCamelCase							:     Optional[Any]	=		os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['''vocab_file'''])
             with open(self.vocab_file      ,							'''w'''      ,							encoding='''utf-8''') as vocab_writer:
                    vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		'''UNwant\u00E9d,running'''
             _lowerCamelCase							:     str	=		'''unwanted, running'''
             return input_text, output_text


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		self.tokenizer_class(self.vocab_file)

             _lowerCamelCase							:     List[str]	=		tokenizer.tokenize('''UNwant\u00E9d,running''')
             self.assertListEqual(a__      ,							['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''])
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)      ,							[7, 4, 5, 10, 8, 9])



      def 				__snake_case			(		self):



             """simple docstring"""





             if not self.test_rust_tokenizer:
                    return

             _lowerCamelCase							:     List[str]	=		self.get_tokenizer()
             _lowerCamelCase							:     str	=		self.get_rust_tokenizer()

             _lowerCamelCase							:     Dict	=		'''I was born in 92000, and this is falsé.'''

             _lowerCamelCase							:     Tuple	=		tokenizer.tokenize(a__)
             _lowerCamelCase							:     Optional[int]	=		rust_tokenizer.tokenize(a__)
             self.assertListEqual(a__      ,							a__)

             _lowerCamelCase							:     Optional[Any]	=		tokenizer.encode(a__      ,							add_special_tokens=a__)
             _lowerCamelCase							:     int	=		rust_tokenizer.encode(a__      ,							add_special_tokens=a__)
             self.assertListEqual(a__      ,							a__)

             _lowerCamelCase							:     Dict	=		self.get_rust_tokenizer()
             _lowerCamelCase							:     List[Any]	=		tokenizer.encode(a__)
             _lowerCamelCase							:     Dict	=		rust_tokenizer.encode(a__)
             self.assertListEqual(a__      ,							a__)








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


_lowerCamelCase			       =      {
    'configuration_layoutlmv3': [
        'LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'LayoutLMv3Config',
        'LayoutLMv3OnnxConfig',
    ],
    'processing_layoutlmv3': ['LayoutLMv3Processor'],
    'tokenization_layoutlmv3': ['LayoutLMv3Tokenizer'],
}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _lowerCamelCase			       =      ['LayoutLMv3TokenizerFast']

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _lowerCamelCase			       =      [
           'LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST',
           'LayoutLMv3ForQuestionAnswering',
           'LayoutLMv3ForSequenceClassification',
           'LayoutLMv3ForTokenClassification',
           'LayoutLMv3Model',
           'LayoutLMv3PreTrainedModel',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _lowerCamelCase			       =      [
           'TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST',
           'TFLayoutLMv3ForQuestionAnswering',
           'TFLayoutLMv3ForSequenceClassification',
           'TFLayoutLMv3ForTokenClassification',
           'TFLayoutLMv3Model',
           'TFLayoutLMv3PreTrainedModel',
       ]

try:
       if not is_vision_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _lowerCamelCase			       =      ['LayoutLMv3FeatureExtractor']
       _lowerCamelCase			       =      ['LayoutLMv3ImageProcessor']


if TYPE_CHECKING:
       from .configuration_layoutlmva import (
           LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP,
           LayoutLMvaConfig,
           LayoutLMvaOnnxConfig,
       )
       from .processing_layoutlmva import LayoutLMvaProcessor
       from .tokenization_layoutlmva import LayoutLMvaTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_layoutlmva import (
                  LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
                  LayoutLMvaForQuestionAnswering,
                  LayoutLMvaForSequenceClassification,
                  LayoutLMvaForTokenClassification,
                  LayoutLMvaModel,
                  LayoutLMvaPreTrainedModel,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_layoutlmva import (
                  TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFLayoutLMvaForQuestionAnswering,
                  TFLayoutLMvaForSequenceClassification,
                  TFLayoutLMvaForTokenClassification,
                  TFLayoutLMvaModel,
                  TFLayoutLMvaPreTrainedModel,
              )

       try:
              if not is_vision_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor
              from .image_processing_layoutlmva import LayoutLMvaImageProcessor

else:
       import sys

       _lowerCamelCase			       =      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

