





import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 			__A       (      lowerCamelCase__ ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      FunnelTokenizer
      UpperCAmelCase__						=      FunnelTokenizerFast
      UpperCAmelCase__						=      True
      UpperCAmelCase__						=      True


      def 				__snake_case			(		self):



             """simple docstring"""





             super().setUp()

             _lowerCamelCase							:     List[Any]	=		[
                 '''<unk>''',
                 '''<cls>''',
                 '''<sep>''',
                 '''want''',
                 '''##want''',
                 '''##ed''',
                 '''wa''',
                 '''un''',
                 '''runn''',
                 '''##ing''',
                 ''',''',
                 '''low''',
                 '''lowest''',
             ]
             _lowerCamelCase							:     Union[str, Any]	=		os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['''vocab_file'''])
             with open(self.vocab_file      ,							'''w'''      ,							encoding='''utf-8''') as vocab_writer:
                    vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))


      def 				__snake_case			(		self      ,							**a__):



             """simple docstring"""





             return FunnelTokenizer.from_pretrained(self.tmpdirname      ,							**a__)


      def 				__snake_case			(		self      ,							**a__):



             """simple docstring"""





             return FunnelTokenizerFast.from_pretrained(self.tmpdirname      ,							**a__)


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		'''UNwant\u00E9d,running'''
             _lowerCamelCase							:     Tuple	=		'''unwanted, running'''
             return input_text, output_text


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		self.tokenizer_class(self.vocab_file)

             _lowerCamelCase							:     Dict	=		tokenizer.tokenize('''UNwant\u00E9d,running''')
             self.assertListEqual(a__      ,							['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''])
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)      ,							[7, 4, 5, 10, 8, 9])



      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		self.get_tokenizers(do_lower_case=a__)
             for tokenizer in tokenizers:
                    _lowerCamelCase							:     Optional[Any]	=		tokenizer('''UNwant\u00E9d,running''')
                    _lowerCamelCase							:     List[Any]	=		len(inputs['''input_ids''']) - 1
                    self.assertListEqual(inputs['''token_type_ids''']      ,							[2] + [0] * sentence_len)

                    _lowerCamelCase							:     str	=		tokenizer('''UNwant\u00E9d,running'''      ,							'''UNwant\u00E9d,running''')
                    self.assertListEqual(inputs['''token_type_ids''']      ,							[2] + [0] * sentence_len + [1] * sentence_len)








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


_lowerCamelCase			       =      {
    'configuration_roberta_prelayernorm': [
        'ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'RobertaPreLayerNormConfig',
        'RobertaPreLayerNormOnnxConfig',
    ],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _lowerCamelCase			       =      [
           'ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST',
           'RobertaPreLayerNormForCausalLM',
           'RobertaPreLayerNormForMaskedLM',
           'RobertaPreLayerNormForMultipleChoice',
           'RobertaPreLayerNormForQuestionAnswering',
           'RobertaPreLayerNormForSequenceClassification',
           'RobertaPreLayerNormForTokenClassification',
           'RobertaPreLayerNormModel',
           'RobertaPreLayerNormPreTrainedModel',
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _lowerCamelCase			       =      [
           'TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST',
           'TFRobertaPreLayerNormForCausalLM',
           'TFRobertaPreLayerNormForMaskedLM',
           'TFRobertaPreLayerNormForMultipleChoice',
           'TFRobertaPreLayerNormForQuestionAnswering',
           'TFRobertaPreLayerNormForSequenceClassification',
           'TFRobertaPreLayerNormForTokenClassification',
           'TFRobertaPreLayerNormMainLayer',
           'TFRobertaPreLayerNormModel',
           'TFRobertaPreLayerNormPreTrainedModel',
       ]

try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       _lowerCamelCase			       =      [
           'FlaxRobertaPreLayerNormForCausalLM',
           'FlaxRobertaPreLayerNormForMaskedLM',
           'FlaxRobertaPreLayerNormForMultipleChoice',
           'FlaxRobertaPreLayerNormForQuestionAnswering',
           'FlaxRobertaPreLayerNormForSequenceClassification',
           'FlaxRobertaPreLayerNormForTokenClassification',
           'FlaxRobertaPreLayerNormModel',
           'FlaxRobertaPreLayerNormPreTrainedModel',
       ]


if TYPE_CHECKING:
       from .configuration_roberta_prelayernorm import (
           ROBERTA_PRELAYERNORM_PRETRAINED_CONFIG_ARCHIVE_MAP,
           RobertaPreLayerNormConfig,
           RobertaPreLayerNormOnnxConfig,
       )

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_roberta_prelayernorm import (
                  ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
                  RobertaPreLayerNormForCausalLM,
                  RobertaPreLayerNormForMaskedLM,
                  RobertaPreLayerNormForMultipleChoice,
                  RobertaPreLayerNormForQuestionAnswering,
                  RobertaPreLayerNormForSequenceClassification,
                  RobertaPreLayerNormForTokenClassification,
                  RobertaPreLayerNormModel,
                  RobertaPreLayerNormPreTrainedModel,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_roberta_prelayernorm import (
                  TF_ROBERTA_PRELAYERNORM_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFRobertaPreLayerNormForCausalLM,
                  TFRobertaPreLayerNormForMaskedLM,
                  TFRobertaPreLayerNormForMultipleChoice,
                  TFRobertaPreLayerNormForQuestionAnswering,
                  TFRobertaPreLayerNormForSequenceClassification,
                  TFRobertaPreLayerNormForTokenClassification,
                  TFRobertaPreLayerNormMainLayer,
                  TFRobertaPreLayerNormModel,
                  TFRobertaPreLayerNormPreTrainedModel,
              )

       try:
              if not is_flax_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_flax_roberta_prelayernorm import (
                  FlaxRobertaPreLayerNormForCausalLM,
                  FlaxRobertaPreLayerNormForMaskedLM,
                  FlaxRobertaPreLayerNormForMultipleChoice,
                  FlaxRobertaPreLayerNormForQuestionAnswering,
                  FlaxRobertaPreLayerNormForSequenceClassification,
                  FlaxRobertaPreLayerNormForTokenClassification,
                  FlaxRobertaPreLayerNormModel,
                  FlaxRobertaPreLayerNormPreTrainedModel,
              )

else:
       import sys

       _lowerCamelCase			       =      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

