

'''simple docstring'''







import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 					A__      (      lowerCAmelCase__						,						unittest.TestCase			):
    lowercase						=		MobileBertTokenizer
    lowercase						=		MobileBertTokenizerFast
    lowercase						=		True
    lowercase						=		True
    lowercase						=		filter_non_english
    lowercase						=		"google/mobilebert-uncased"



    def 					snake_case_			(					self       )						->      Optional[int]:





           '''simple docstring'''
           super().setUp()

           A_						=      [
               """[UNK]""",
               """[CLS]""",
               """[SEP]""",
               """[PAD]""",
               """[MASK]""",
               """want""",
               """##want""",
               """##ed""",
               """wa""",
               """un""",
               """runn""",
               """##ing""",
               """,""",
               """low""",
               """lowest""",
           ]
           A_						=      os.path.join(self.tmpdirname		,				VOCAB_FILES_NAMES["""vocab_file"""]       )
           with open(self.vocab_file		,				"""w"""		,				encoding="""utf-8"""       ) as vocab_writer:
                  vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )

           A_						=      [
               (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
               for tokenizer_def in self.tokenizers_list
           ]



    def 					snake_case_			(					self		,				UpperCamelCase__       )						->      int:





           '''simple docstring'''
           A_						=      """UNwant\u00E9d,running"""
           A_						=      """unwanted, running"""
           return input_text, output_text



    def 					snake_case_			(					self       )						->      Optional[Any]:





           '''simple docstring'''
           A_						=      self.tokenizer_class(self.vocab_file       )

           A_						=      tokenizer.tokenize("""UNwant\u00E9d,running"""       )
           self.assertListEqual(_lowerCamelCase		,				["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]       )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase       )		,				[9, 6, 7, 12, 10, 11]       )



    def 					snake_case_			(					self       )						->      Optional[int]:





           '''simple docstring'''





           if not self.test_rust_tokenizer:
                  return

           A_						=      self.get_tokenizer()
           A_						=      self.get_rust_tokenizer()

           A_						=      """UNwant\u00E9d,running"""

           A_						=      tokenizer.tokenize(_lowerCamelCase       )
           A_						=      rust_tokenizer.tokenize(_lowerCamelCase       )
           self.assertListEqual(_lowerCamelCase		,				_lowerCamelCase       )

           A_						=      tokenizer.encode(_lowerCamelCase		,				add_special_tokens=_lowerCamelCase       )
           A_						=      rust_tokenizer.encode(_lowerCamelCase		,				add_special_tokens=_lowerCamelCase       )
           self.assertListEqual(_lowerCamelCase		,				_lowerCamelCase       )

           A_						=      self.get_rust_tokenizer()
           A_						=      tokenizer.encode(_lowerCamelCase       )
           A_						=      rust_tokenizer.encode(_lowerCamelCase       )
           self.assertListEqual(_lowerCamelCase		,				_lowerCamelCase       )

           # With lower casing
           A_						=      self.get_tokenizer(do_lower_case=_lowerCamelCase       )
           A_						=      self.get_rust_tokenizer(do_lower_case=_lowerCamelCase       )

           A_						=      """UNwant\u00E9d,running"""

           A_						=      tokenizer.tokenize(_lowerCamelCase       )
           A_						=      rust_tokenizer.tokenize(_lowerCamelCase       )
           self.assertListEqual(_lowerCamelCase		,				_lowerCamelCase       )

           A_						=      tokenizer.encode(_lowerCamelCase		,				add_special_tokens=_lowerCamelCase       )
           A_						=      rust_tokenizer.encode(_lowerCamelCase		,				add_special_tokens=_lowerCamelCase       )
           self.assertListEqual(_lowerCamelCase		,				_lowerCamelCase       )

           A_						=      self.get_rust_tokenizer()
           A_						=      tokenizer.encode(_lowerCamelCase       )
           A_						=      rust_tokenizer.encode(_lowerCamelCase       )
           self.assertListEqual(_lowerCamelCase		,				_lowerCamelCase       )



    def 					snake_case_			(					self       )						->      Tuple:





           '''simple docstring'''
           A_						=      BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""       )		,				["""ah""", """\u535A""", """\u63A8""", """zz"""]       )



    def 					snake_case_			(					self       )						->      int:





           '''simple docstring'''
           A_						=      BasicTokenizer(do_lower_case=_lowerCamelCase       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       )		,				["""hello""", """!""", """how""", """are""", """you""", """?"""]       )
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )		,				["""hello"""]       )



    def 					snake_case_			(					self       )						->      List[Any]:





           '''simple docstring'''
           A_						=      BasicTokenizer(do_lower_case=_lowerCamelCase		,				strip_accents=_lowerCamelCase       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )		,				["""hällo""", """!""", """how""", """are""", """you""", """?"""]       )
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )		,				["""h\u00E9llo"""]       )



    def 					snake_case_			(					self       )						->      Tuple:





           '''simple docstring'''
           A_						=      BasicTokenizer(do_lower_case=_lowerCamelCase		,				strip_accents=_lowerCamelCase       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )		,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )		,				["""hello"""]       )



    def 					snake_case_			(					self       )						->      str:





           '''simple docstring'''
           A_						=      BasicTokenizer(do_lower_case=_lowerCamelCase       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )		,				["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )		,				["""hello"""]       )



    def 					snake_case_			(					self       )						->      Optional[Any]:





           '''simple docstring'''
           A_						=      BasicTokenizer(do_lower_case=_lowerCamelCase       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       )		,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )



    def 					snake_case_			(					self       )						->      List[Any]:





           '''simple docstring'''
           A_						=      BasicTokenizer(do_lower_case=_lowerCamelCase		,				strip_accents=_lowerCamelCase       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )		,				["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )



    def 					snake_case_			(					self       )						->      Optional[Any]:





           '''simple docstring'''
           A_						=      BasicTokenizer(do_lower_case=_lowerCamelCase		,				strip_accents=_lowerCamelCase       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )		,				["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )



    def 					snake_case_			(					self       )						->      Dict:





           '''simple docstring'''
           A_						=      BasicTokenizer(do_lower_case=_lowerCamelCase		,				never_split=["""[UNK]"""]       )

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""       )		,				["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]       )



    def 					snake_case_			(					self       )						->      int:





           '''simple docstring'''
           A_						=      ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

           A_						=      {}
           for i, token in enumerate(_lowerCamelCase       ):
                  A_						=      i
           A_						=      WordpieceTokenizer(vocab=_lowerCamelCase		,				unk_token="""[UNK]"""       )

           self.assertListEqual(tokenizer.tokenize(""""""       )		,				[]       )

           self.assertListEqual(tokenizer.tokenize("""unwanted running"""       )		,				["""un""", """##want""", """##ed""", """runn""", """##ing"""]       )

           self.assertListEqual(tokenizer.tokenize("""unwantedX running"""       )		,				["""[UNK]""", """runn""", """##ing"""]       )



    def 					snake_case_			(					self       )						->      List[str]:





           '''simple docstring'''
           self.assertTrue(_is_whitespace(""" """       )       )
           self.assertTrue(_is_whitespace("""\t"""       )       )
           self.assertTrue(_is_whitespace("""\r"""       )       )
           self.assertTrue(_is_whitespace("""\n"""       )       )
           self.assertTrue(_is_whitespace("""\u00A0"""       )       )

           self.assertFalse(_is_whitespace("""A"""       )       )
           self.assertFalse(_is_whitespace("""-"""       )       )



    def 					snake_case_			(					self       )						->      Dict:





           '''simple docstring'''
           self.assertTrue(_is_control("""\u0005"""       )       )

           self.assertFalse(_is_control("""A"""       )       )
           self.assertFalse(_is_control(""" """       )       )
           self.assertFalse(_is_control("""\t"""       )       )
           self.assertFalse(_is_control("""\r"""       )       )



    def 					snake_case_			(					self       )						->      Union[str, Any]:





           '''simple docstring'''
           self.assertTrue(_is_punctuation("""-"""       )       )
           self.assertTrue(_is_punctuation("""$"""       )       )
           self.assertTrue(_is_punctuation("""`"""       )       )
           self.assertTrue(_is_punctuation("""."""       )       )

           self.assertFalse(_is_punctuation("""A"""       )       )
           self.assertFalse(_is_punctuation(""" """       )       )



    def 					snake_case_			(					self       )						->      Tuple:





           '''simple docstring'''
           A_						=      self.get_tokenizer()
           A_						=      self.get_rust_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(_lowerCamelCase       ) for t in ["""Test""", """\xad""", """test"""]]		,				[["""[UNK]"""], [], ["""[UNK]"""]]       )

           self.assertListEqual(
               [rust_tokenizer.tokenize(_lowerCamelCase       ) for t in ["""Test""", """\xad""", """test"""]]		,				[["""[UNK]"""], [], ["""[UNK]"""]]       )



    @slow
    def 					snake_case_			(					self       )						->      List[str]:





           '''simple docstring'''
           A_						=      self.tokenizer_class.from_pretrained("""google/mobilebert-uncased"""       )

           A_						=      tokenizer.encode("""sequence builders"""		,				add_special_tokens=_lowerCamelCase       )
           A_						=      tokenizer.encode("""multi-sequence build"""		,				add_special_tokens=_lowerCamelCase       )

           A_						=      tokenizer.build_inputs_with_special_tokens(_lowerCamelCase       )
           A_						=      tokenizer.build_inputs_with_special_tokens(_lowerCamelCase		,				_lowerCamelCase       )

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]



    def 					snake_case_			(					self       )						->      List[Any]:





           '''simple docstring'''





           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                         A_						=      self.rust_tokenizer_class.from_pretrained(_lowerCamelCase		,				**_lowerCamelCase       )

                         A_						=      f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                         A_						=      tokenizer_r.encode_plus(
                             _lowerCamelCase		,				return_attention_mask=_lowerCamelCase		,				return_token_type_ids=_lowerCamelCase		,				return_offsets_mapping=_lowerCamelCase		,				add_special_tokens=_lowerCamelCase		,				)

                         A_						=      tokenizer_r.do_lower_case if hasattr(_lowerCamelCase		,				"""do_lower_case"""       ) else False
                         A_						=      (
                             [
                                 ((0, 0), tokenizer_r.cls_token),
                                 ((0, 1), """A"""),
                                 ((1, 2), ""","""),
                                 ((3, 5), """na"""),
                                 ((5, 6), """##ï"""),
                                 ((6, 8), """##ve"""),
                                 ((9, 15), tokenizer_r.mask_token),
                                 ((16, 21), """Allen"""),
                                 ((21, 23), """##NL"""),
                                 ((23, 24), """##P"""),
                                 ((25, 33), """sentence"""),
                                 ((33, 34), """."""),
                                 ((0, 0), tokenizer_r.sep_token),
                             ]
                             if not do_lower_case
                             else [
                                 ((0, 0), tokenizer_r.cls_token),
                                 ((0, 1), """a"""),
                                 ((1, 2), ""","""),
                                 ((3, 8), """naive"""),
                                 ((9, 15), tokenizer_r.mask_token),
                                 ((16, 21), """allen"""),
                                 ((21, 23), """##nl"""),
                                 ((23, 24), """##p"""),
                                 ((25, 33), """sentence"""),
                                 ((33, 34), """."""),
                                 ((0, 0), tokenizer_r.sep_token),
                             ]
                         )

                         self.assertEqual(
                             [e[1] for e in expected_results]		,				tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]       )       )
                         self.assertEqual([e[0] for e in expected_results]		,				tokens["""offset_mapping"""]       )







    def 					snake_case_			(					self       )						->      List[Any]:





           '''simple docstring'''
           A_						=      ["""的""", """人""", """有"""]
           A_						=      """""".join(_lowerCamelCase       )
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
                         A_						=      True
                         A_						=      self.tokenizer_class.from_pretrained(_lowerCamelCase		,				**_lowerCamelCase       )
                         A_						=      self.rust_tokenizer_class.from_pretrained(_lowerCamelCase		,				**_lowerCamelCase       )

                         A_						=      tokenizer_p.encode(_lowerCamelCase		,				add_special_tokens=_lowerCamelCase       )
                         A_						=      tokenizer_r.encode(_lowerCamelCase		,				add_special_tokens=_lowerCamelCase       )

                         A_						=      tokenizer_r.convert_ids_to_tokens(_lowerCamelCase       )
                         A_						=      tokenizer_p.convert_ids_to_tokens(_lowerCamelCase       )

                         # it is expected that each Chinese character is not preceded by "##"
                         self.assertListEqual(_lowerCamelCase		,				_lowerCamelCase       )
                         self.assertListEqual(_lowerCamelCase		,				_lowerCamelCase       )

                         A_						=      False
                         A_						=      self.rust_tokenizer_class.from_pretrained(_lowerCamelCase		,				**_lowerCamelCase       )
                         A_						=      self.tokenizer_class.from_pretrained(_lowerCamelCase		,				**_lowerCamelCase       )

                         A_						=      tokenizer_r.encode(_lowerCamelCase		,				add_special_tokens=_lowerCamelCase       )
                         A_						=      tokenizer_p.encode(_lowerCamelCase		,				add_special_tokens=_lowerCamelCase       )

                         A_						=      tokenizer_r.convert_ids_to_tokens(_lowerCamelCase       )
                         A_						=      tokenizer_p.convert_ids_to_tokens(_lowerCamelCase       )

                         # it is expected that only the first Chinese character is not preceded by "##".
                         A_						=      [
                             f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowerCamelCase       )
                         ]
                         self.assertListEqual(_lowerCamelCase		,				_lowerCamelCase       )
                         self.assertListEqual(_lowerCamelCase		,				_lowerCamelCase       )



'''simple docstring'''







def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__,       UpperCAmelCase__ = 0,       UpperCAmelCase__ = 0				)			->			int:
       A_						=      right or len(UpperCAmelCase__				) - 1
       if left > right:
              return -1
       elif list_data[left] == key:
              return left
       elif list_data[right] == key:
              return right
       else:
              return search(UpperCAmelCase__,       UpperCAmelCase__,       left + 1,       right - 1				)


if __name__ == "__main__":
      import doctest

      doctest.testmod()
