

import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class      UpperCamelCase (     SCREAMING_SNAKE_CASE	,			unittest.TestCase   ):
    __UpperCamelCase         =BertTokenizer
    __UpperCamelCase         =BertTokenizerFast
    __UpperCamelCase         =True
    __UpperCamelCase         =True
    __UpperCamelCase         =filter_non_english

    def UpperCamelCase			(			self :      Any    ):

     """simple docstring"""






     super().setUp()

     SCREAMING_SNAKE_CASE								=			[
         '[UNK]',
         '[CLS]',
         '[SEP]',
         '[PAD]',
         '[MASK]',
         'want',
         '##want',
         '##ed',
         'wa',
         'un',
         'runn',
         '##ing',
         ',',
         'low',
         'lowest',
     ]
     SCREAMING_SNAKE_CASE								=			os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES['vocab_file']    )
     with open(self.vocab_file							,    'w'							,    encoding='utf-8'    ) as vocab_writer:
      vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )

    def UpperCamelCase			(			self :      Any							,    snake_case__ :      int    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			'UNwant\u00E9d,running'
     SCREAMING_SNAKE_CASE								=			'unwanted, running'
     return input_text, output_text

    def UpperCamelCase			(			self :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.tokenizer_class(self.vocab_file    )

     SCREAMING_SNAKE_CASE								=			tokenizer.tokenize('UNwant\u00E9d,running'    )
     self.assertListEqual(snake_case__							,    ['un', '##want', '##ed', ',', 'runn', '##ing']    )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__    )							,    [9, 6, 7, 1_2, 1_0, 1_1]    )

    def UpperCamelCase			(			self :      str    ):

     """simple docstring"""






     if not self.test_rust_tokenizer:
      return

     SCREAMING_SNAKE_CASE								=			self.get_tokenizer()
     SCREAMING_SNAKE_CASE								=			self.get_rust_tokenizer()

     SCREAMING_SNAKE_CASE								=			'UNwant\u00E9d,running'

     SCREAMING_SNAKE_CASE								=			tokenizer.tokenize(snake_case__    )
     SCREAMING_SNAKE_CASE								=			rust_tokenizer.tokenize(snake_case__    )
     self.assertListEqual(snake_case__							,    snake_case__    )

     SCREAMING_SNAKE_CASE								=			tokenizer.encode(snake_case__							,    add_special_tokens=snake_case__    )
     SCREAMING_SNAKE_CASE								=			rust_tokenizer.encode(snake_case__							,    add_special_tokens=snake_case__    )
     self.assertListEqual(snake_case__							,    snake_case__    )

     SCREAMING_SNAKE_CASE								=			self.get_rust_tokenizer()
     SCREAMING_SNAKE_CASE								=			tokenizer.encode(snake_case__    )
     SCREAMING_SNAKE_CASE								=			rust_tokenizer.encode(snake_case__    )
     self.assertListEqual(snake_case__							,    snake_case__    )

     # With lower casing
     SCREAMING_SNAKE_CASE								=			self.get_tokenizer(do_lower_case=snake_case__    )
     SCREAMING_SNAKE_CASE								=			self.get_rust_tokenizer(do_lower_case=snake_case__    )

     SCREAMING_SNAKE_CASE								=			'UNwant\u00E9d,running'

     SCREAMING_SNAKE_CASE								=			tokenizer.tokenize(snake_case__    )
     SCREAMING_SNAKE_CASE								=			rust_tokenizer.tokenize(snake_case__    )
     self.assertListEqual(snake_case__							,    snake_case__    )

     SCREAMING_SNAKE_CASE								=			tokenizer.encode(snake_case__							,    add_special_tokens=snake_case__    )
     SCREAMING_SNAKE_CASE								=			rust_tokenizer.encode(snake_case__							,    add_special_tokens=snake_case__    )
     self.assertListEqual(snake_case__							,    snake_case__    )

     SCREAMING_SNAKE_CASE								=			self.get_rust_tokenizer()
     SCREAMING_SNAKE_CASE								=			tokenizer.encode(snake_case__    )
     SCREAMING_SNAKE_CASE								=			rust_tokenizer.encode(snake_case__    )
     self.assertListEqual(snake_case__							,    snake_case__    )

    def UpperCamelCase			(			self :      Tuple    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'    )							,    ['ah', '\u535A', '\u63A8', 'zz']    )

    def UpperCamelCase			(			self :      Optional[int]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BasicTokenizer(do_lower_case=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )							,    ['hello', '!', 'how', 'are', 'you', '?']    )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )							,    ['hello']    )

    def UpperCamelCase			(			self :      Optional[int]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BasicTokenizer(do_lower_case=snake_case__							,    strip_accents=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )							,    ['hällo', '!', 'how', 'are', 'you', '?']    )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )							,    ['h\u00E9llo']    )

    def UpperCamelCase			(			self :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BasicTokenizer(do_lower_case=snake_case__							,    strip_accents=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )							,    ['hallo', '!', 'how', 'are', 'you', '?']    )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )							,    ['hello']    )

    def UpperCamelCase			(			self :      Tuple    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BasicTokenizer(do_lower_case=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )							,    ['hallo', '!', 'how', 'are', 'you', '?']    )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )							,    ['hello']    )

    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BasicTokenizer(do_lower_case=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )							,    ['HeLLo', '!', 'how', 'Are', 'yoU', '?']    )

    def UpperCamelCase			(			self :      Union[str, Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BasicTokenizer(do_lower_case=snake_case__							,    strip_accents=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )							,    ['HäLLo', '!', 'how', 'Are', 'yoU', '?']    )

    def UpperCamelCase			(			self :      Union[str, Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BasicTokenizer(do_lower_case=snake_case__							,    strip_accents=snake_case__    )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )							,    ['HaLLo', '!', 'how', 'Are', 'yoU', '?']    )

    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BasicTokenizer(do_lower_case=snake_case__							,    never_split=['[UNK]']    )

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'    )							,    ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']    )

    def UpperCamelCase			(			self :      Dict    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			BasicTokenizer()
     SCREAMING_SNAKE_CASE								=			'a\n\'ll !!to?\'d of, can\'t.'
     SCREAMING_SNAKE_CASE								=			['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
     self.assertListEqual(tokenizer.tokenize(snake_case__    )							,    snake_case__    )

    def UpperCamelCase			(			self :      int    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

     SCREAMING_SNAKE_CASE								=			{}
     for i, token in enumerate(snake_case__    ):
      SCREAMING_SNAKE_CASE								=			i
     SCREAMING_SNAKE_CASE								=			WordpieceTokenizer(vocab=snake_case__							,    unk_token='[UNK]'    )

     self.assertListEqual(tokenizer.tokenize(''    )							,    []    )

     self.assertListEqual(tokenizer.tokenize('unwanted running'    )							,    ['un', '##want', '##ed', 'runn', '##ing']    )

     self.assertListEqual(tokenizer.tokenize('unwantedX running'    )							,    ['[UNK]', 'runn', '##ing']    )

    def UpperCamelCase			(			self :      Tuple    ):

     """simple docstring"""






     self.assertTrue(_is_whitespace(' '    )    )
     self.assertTrue(_is_whitespace('\t'    )    )
     self.assertTrue(_is_whitespace('\r'    )    )
     self.assertTrue(_is_whitespace('\n'    )    )
     self.assertTrue(_is_whitespace('\u00A0'    )    )

     self.assertFalse(_is_whitespace('A'    )    )
     self.assertFalse(_is_whitespace('-'    )    )

    def UpperCamelCase			(			self :      Optional[int]    ):

     """simple docstring"""






     self.assertTrue(_is_control('\u0005'    )    )

     self.assertFalse(_is_control('A'    )    )
     self.assertFalse(_is_control(' '    )    )
     self.assertFalse(_is_control('\t'    )    )
     self.assertFalse(_is_control('\r'    )    )

    def UpperCamelCase			(			self :      List[Any]    ):

     """simple docstring"""






     self.assertTrue(_is_punctuation('-'    )    )
     self.assertTrue(_is_punctuation('$'    )    )
     self.assertTrue(_is_punctuation('`'    )    )
     self.assertTrue(_is_punctuation('.'    )    )

     self.assertFalse(_is_punctuation('A'    )    )
     self.assertFalse(_is_punctuation(' '    )    )

    def UpperCamelCase			(			self :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.get_tokenizer()
     SCREAMING_SNAKE_CASE								=			self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(snake_case__    ) for t in ['Test', '\xad', 'test']]							,    [['[UNK]'], [], ['[UNK]']]    )

     self.assertListEqual(
         [rust_tokenizer.tokenize(snake_case__    ) for t in ['Test', '\xad', 'test']]							,    [['[UNK]'], [], ['[UNK]']]    )

    @slow
    def UpperCamelCase			(			self :      List[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.tokenizer_class.from_pretrained('bert-base-uncased'    )

     SCREAMING_SNAKE_CASE								=			tokenizer.encode('sequence builders'							,    add_special_tokens=snake_case__    )
     SCREAMING_SNAKE_CASE								=			tokenizer.encode('multi-sequence build'							,    add_special_tokens=snake_case__    )

     SCREAMING_SNAKE_CASE								=			tokenizer.build_inputs_with_special_tokens(snake_case__    )
     SCREAMING_SNAKE_CASE								=			tokenizer.build_inputs_with_special_tokens(snake_case__							,    snake_case__    )

     assert encoded_sentence == [1_0_1] + text + [1_0_2]
     assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]

    def UpperCamelCase			(			self :      Optional[int]    ):

     """simple docstring"""






     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
       SCREAMING_SNAKE_CASE								=			self.rust_tokenizer_class.from_pretrained(snake_case__							,    **snake_case__    )

       SCREAMING_SNAKE_CASE								=			F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
       SCREAMING_SNAKE_CASE								=			tokenizer_r.encode_plus(
           snake_case__							,    return_attention_mask=snake_case__							,    return_token_type_ids=snake_case__							,    return_offsets_mapping=snake_case__							,    add_special_tokens=snake_case__							,    )

       SCREAMING_SNAKE_CASE								=			tokenizer_r.do_lower_case if hasattr(snake_case__							,    'do_lower_case'    ) else False
       SCREAMING_SNAKE_CASE								=			(
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), 'A'),
               ((1, 2), ','),
               ((3, 5), 'na'),
               ((5, 6), '##ï'),
               ((6, 8), '##ve'),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), 'Allen'),
               ((2_1, 2_3), '##NL'),
               ((2_3, 2_4), '##P'),
               ((2_5, 3_3), 'sentence'),
               ((3_3, 3_4), '.'),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), 'a'),
               ((1, 2), ','),
               ((3, 8), 'naive'),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), 'allen'),
               ((2_1, 2_3), '##nl'),
               ((2_3, 2_4), '##p'),
               ((2_5, 3_3), 'sentence'),
               ((3_3, 3_4), '.'),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]							,    tokenizer_r.convert_ids_to_tokens(tokens['input_ids']    )    )
       self.assertEqual([e[0] for e in expected_results]							,    tokens['offset_mapping']    )




    def UpperCamelCase			(			self :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			['的', '人', '有']
     SCREAMING_SNAKE_CASE								=			''.join(snake_case__    )
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
       SCREAMING_SNAKE_CASE								=			True
       SCREAMING_SNAKE_CASE								=			self.tokenizer_class.from_pretrained(snake_case__							,    **snake_case__    )
       SCREAMING_SNAKE_CASE								=			self.rust_tokenizer_class.from_pretrained(snake_case__							,    **snake_case__    )

       SCREAMING_SNAKE_CASE								=			tokenizer_p.encode(snake_case__							,    add_special_tokens=snake_case__    )
       SCREAMING_SNAKE_CASE								=			tokenizer_r.encode(snake_case__							,    add_special_tokens=snake_case__    )

       SCREAMING_SNAKE_CASE								=			tokenizer_r.convert_ids_to_tokens(snake_case__    )
       SCREAMING_SNAKE_CASE								=			tokenizer_p.convert_ids_to_tokens(snake_case__    )

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(snake_case__							,    snake_case__    )
       self.assertListEqual(snake_case__							,    snake_case__    )

       SCREAMING_SNAKE_CASE								=			False
       SCREAMING_SNAKE_CASE								=			self.rust_tokenizer_class.from_pretrained(snake_case__							,    **snake_case__    )
       SCREAMING_SNAKE_CASE								=			self.tokenizer_class.from_pretrained(snake_case__							,    **snake_case__    )

       SCREAMING_SNAKE_CASE								=			tokenizer_r.encode(snake_case__							,    add_special_tokens=snake_case__    )
       SCREAMING_SNAKE_CASE								=			tokenizer_p.encode(snake_case__							,    add_special_tokens=snake_case__    )

       SCREAMING_SNAKE_CASE								=			tokenizer_r.convert_ids_to_tokens(snake_case__    )
       SCREAMING_SNAKE_CASE								=			tokenizer_p.convert_ids_to_tokens(snake_case__    )

       # it is expected that only the first Chinese character is not preceded by "##".
       SCREAMING_SNAKE_CASE								=			[
           F"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case__    )
       ]
       self.assertListEqual(snake_case__							,    snake_case__    )
       self.assertListEqual(snake_case__							,    snake_case__    )




from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
      if not (is_transformers_available() and is_torch_available()):
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      from ...utils.dummy_torch_and_transformers_objects import (
          ImageTextPipelineOutput,
          UniDiffuserPipeline,
      )
else:
      from .modeling_text_decoder import UniDiffuserTextDecoder
      from .modeling_uvit import UniDiffuserModel, UTransformeraDModel
      from .pipeline_unidiffuser import ImageTextPipelineOutput, UniDiffuserPipeline

