"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class  _snake_case		(    __snake_case   ,       unittest.TestCase ):







				"""simple docstring"""






				a			=						MobileBertTokenizer
				a			=						MobileBertTokenizerFast
				a			=						True
				a			=						True
				a			=						filter_non_english
				a			=						"google/mobilebert-uncased"





				def 		_lowerCAmelCase		(      self							:							List[Any]):



											"""simple docstring"""







											super().setUp()

											_SCREAMING_SNAKE_CASE		:						Tuple			  =    [
											    """[UNK]""",
											    """[CLS]""",
											    """[SEP]""",
											    """[PAD]""",
											    """[MASK]""",
											    """want""",
											    """##want""",
											    """##ed""",
											    """wa""",
											    """un""",
											    """runn""",
											    """##ing""",
											    """,""",
											    """low""",
											    """lowest""",
											]
											_SCREAMING_SNAKE_CASE		:						Optional[Any]			  =    os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES["""vocab_file"""])
											with open(self.vocab_file      ,    """w"""      ,    encoding="""utf-8""") as vocab_writer:
																		vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))

											_SCREAMING_SNAKE_CASE		:						Any			  =    [
											    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
											    for tokenizer_def in self.tokenizers_list
											]





				def 		_lowerCAmelCase		(      self							:							Tuple      ,    _A							:							str):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						Dict			  =    """UNwant\u00E9d,running"""
											_SCREAMING_SNAKE_CASE		:						Any			  =    """unwanted, running"""
											return input_text, output_text





				def 		_lowerCAmelCase		(      self							:							Optional[int]):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						Any			  =    self.tokenizer_class(self.vocab_file)

											_SCREAMING_SNAKE_CASE		:						Optional[Any]			  =    tokenizer.tokenize("""UNwant\u00E9d,running""")
											self.assertListEqual(_A      ,    ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""])
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_A)      ,    [9, 6, 7, 1_2, 1_0, 1_1])





				def 		_lowerCAmelCase		(      self							:							Union[str, Any]):



											"""simple docstring"""







											if not self.test_rust_tokenizer:
																		return

											_SCREAMING_SNAKE_CASE		:						List[Any]			  =    self.get_tokenizer()
											_SCREAMING_SNAKE_CASE		:						Optional[int]			  =    self.get_rust_tokenizer()

											_SCREAMING_SNAKE_CASE		:						int			  =    """UNwant\u00E9d,running"""

											_SCREAMING_SNAKE_CASE		:						List[str]			  =    tokenizer.tokenize(_A)
											_SCREAMING_SNAKE_CASE		:						Dict			  =    rust_tokenizer.tokenize(_A)
											self.assertListEqual(_A      ,    _A)

											_SCREAMING_SNAKE_CASE		:						Optional[int]			  =    tokenizer.encode(_A      ,    add_special_tokens=_A)
											_SCREAMING_SNAKE_CASE		:						Union[str, Any]			  =    rust_tokenizer.encode(_A      ,    add_special_tokens=_A)
											self.assertListEqual(_A      ,    _A)

											_SCREAMING_SNAKE_CASE		:						Optional[Any]			  =    self.get_rust_tokenizer()
											_SCREAMING_SNAKE_CASE		:						List[Any]			  =    tokenizer.encode(_A)
											_SCREAMING_SNAKE_CASE		:						Tuple			  =    rust_tokenizer.encode(_A)
											self.assertListEqual(_A      ,    _A)

											# With lower casing
											_SCREAMING_SNAKE_CASE		:						Dict			  =    self.get_tokenizer(do_lower_case=_A)
											_SCREAMING_SNAKE_CASE		:						int			  =    self.get_rust_tokenizer(do_lower_case=_A)

											_SCREAMING_SNAKE_CASE		:						Union[str, Any]			  =    """UNwant\u00E9d,running"""

											_SCREAMING_SNAKE_CASE		:						Tuple			  =    tokenizer.tokenize(_A)
											_SCREAMING_SNAKE_CASE		:						Tuple			  =    rust_tokenizer.tokenize(_A)
											self.assertListEqual(_A      ,    _A)

											_SCREAMING_SNAKE_CASE		:						Tuple			  =    tokenizer.encode(_A      ,    add_special_tokens=_A)
											_SCREAMING_SNAKE_CASE		:						Union[str, Any]			  =    rust_tokenizer.encode(_A      ,    add_special_tokens=_A)
											self.assertListEqual(_A      ,    _A)

											_SCREAMING_SNAKE_CASE		:						List[str]			  =    self.get_rust_tokenizer()
											_SCREAMING_SNAKE_CASE		:						int			  =    tokenizer.encode(_A)
											_SCREAMING_SNAKE_CASE		:						Union[str, Any]			  =    rust_tokenizer.encode(_A)
											self.assertListEqual(_A      ,    _A)





				def 		_lowerCAmelCase		(      self							:							Optional[Any]):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						List[str]			  =    BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""")      ,    ["""ah""", """\u535A""", """\u63A8""", """zz"""])





				def 		_lowerCAmelCase		(      self							:							int):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						Union[str, Any]			  =    BasicTokenizer(do_lower_case=_A)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)      ,    ["""hello""", """!""", """how""", """are""", """you""", """?"""])
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")      ,    ["""hello"""])





				def 		_lowerCAmelCase		(      self							:							List[str]):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						Any			  =    BasicTokenizer(do_lower_case=_A      ,    strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)      ,    ["""hällo""", """!""", """how""", """are""", """you""", """?"""])
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")      ,    ["""h\u00E9llo"""])





				def 		_lowerCAmelCase		(      self							:							Dict):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						int			  =    BasicTokenizer(do_lower_case=_A      ,    strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)      ,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""])
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")      ,    ["""hello"""])





				def 		_lowerCAmelCase		(      self							:							str):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						str			  =    BasicTokenizer(do_lower_case=_A)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)      ,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""])
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")      ,    ["""hello"""])





				def 		_lowerCAmelCase		(      self							:							Union[str, Any]):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						List[str]			  =    BasicTokenizer(do_lower_case=_A)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)      ,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""])





				def 		_lowerCAmelCase		(      self							:							Dict):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						Any			  =    BasicTokenizer(do_lower_case=_A      ,    strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)      ,    ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""])





				def 		_lowerCAmelCase		(      self							:							Dict):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						int			  =    BasicTokenizer(do_lower_case=_A      ,    strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)      ,    ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""])





				def 		_lowerCAmelCase		(      self							:							int):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						Tuple			  =    BasicTokenizer(do_lower_case=_A      ,    never_split=["""[UNK]"""])

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""")      ,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""])





				def 		_lowerCAmelCase		(      self							:							Union[str, Any]):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						Optional[int]			  =    ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

											_SCREAMING_SNAKE_CASE		:						str			  =    {}
											for i, token in enumerate(_A):
																		_SCREAMING_SNAKE_CASE		:						Dict			  =    i
											_SCREAMING_SNAKE_CASE		:						Optional[int]			  =    WordpieceTokenizer(vocab=_A      ,    unk_token="""[UNK]""")

											self.assertListEqual(tokenizer.tokenize("""""")      ,    [])

											self.assertListEqual(tokenizer.tokenize("""unwanted running""")      ,    ["""un""", """##want""", """##ed""", """runn""", """##ing"""])

											self.assertListEqual(tokenizer.tokenize("""unwantedX running""")      ,    ["""[UNK]""", """runn""", """##ing"""])





				def 		_lowerCAmelCase		(      self							:							Optional[Any]):



											"""simple docstring"""







											self.assertTrue(_is_whitespace(""" """))
											self.assertTrue(_is_whitespace("""\t"""))
											self.assertTrue(_is_whitespace("""\r"""))
											self.assertTrue(_is_whitespace("""\n"""))
											self.assertTrue(_is_whitespace("""\u00A0"""))

											self.assertFalse(_is_whitespace("""A"""))
											self.assertFalse(_is_whitespace("""-"""))





				def 		_lowerCAmelCase		(      self							:							Optional[int]):



											"""simple docstring"""







											self.assertTrue(_is_control("""\u0005"""))

											self.assertFalse(_is_control("""A"""))
											self.assertFalse(_is_control(""" """))
											self.assertFalse(_is_control("""\t"""))
											self.assertFalse(_is_control("""\r"""))





				def 		_lowerCAmelCase		(      self							:							List[Any]):



											"""simple docstring"""







											self.assertTrue(_is_punctuation("""-"""))
											self.assertTrue(_is_punctuation("""$"""))
											self.assertTrue(_is_punctuation("""`"""))
											self.assertTrue(_is_punctuation("""."""))

											self.assertFalse(_is_punctuation("""A"""))
											self.assertFalse(_is_punctuation(""" """))





				def 		_lowerCAmelCase		(      self							:							Optional[int]):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						List[Any]			  =    self.get_tokenizer()
											_SCREAMING_SNAKE_CASE		:						Union[str, Any]			  =    self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(_A) for t in ["""Test""", """\xad""", """test"""]]      ,    [["""[UNK]"""], [], ["""[UNK]"""]])

											self.assertListEqual(
											    [rust_tokenizer.tokenize(_A) for t in ["""Test""", """\xad""", """test"""]]      ,    [["""[UNK]"""], [], ["""[UNK]"""]])





				@slow
				def 		_lowerCAmelCase		(      self							:							Dict):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						List[str]			  =    self.tokenizer_class.from_pretrained("""google/mobilebert-uncased""")

											_SCREAMING_SNAKE_CASE		:						str			  =    tokenizer.encode("""sequence builders"""      ,    add_special_tokens=_A)
											_SCREAMING_SNAKE_CASE		:						int			  =    tokenizer.encode("""multi-sequence build"""      ,    add_special_tokens=_A)

											_SCREAMING_SNAKE_CASE		:						Optional[Any]			  =    tokenizer.build_inputs_with_special_tokens(_A)
											_SCREAMING_SNAKE_CASE		:						int			  =    tokenizer.build_inputs_with_special_tokens(_A      ,    _A)

											assert encoded_sentence == [1_0_1] + text + [1_0_2]
											assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]





				def 		_lowerCAmelCase		(      self							:							List[str]):



											"""simple docstring"""







											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
																									_SCREAMING_SNAKE_CASE		:						Dict			  =    self.rust_tokenizer_class.from_pretrained(_A      ,    **_A)

																									_SCREAMING_SNAKE_CASE		:						Optional[Any]			  =    f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																									_SCREAMING_SNAKE_CASE		:						Any			  =    tokenizer_r.encode_plus(
																									    _A      ,    return_attention_mask=_A      ,    return_token_type_ids=_A      ,    return_offsets_mapping=_A      ,    add_special_tokens=_A      ,    )

																									_SCREAMING_SNAKE_CASE		:						int			  =    tokenizer_r.do_lower_case if hasattr(_A      ,    """do_lower_case""") else False
																									_SCREAMING_SNAKE_CASE		:						Any			  =    (
																									    [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), """A"""),
																									        ((1, 2), ""","""),
																									        ((3, 5), """na"""),
																									        ((5, 6), """##ï"""),
																									        ((6, 8), """##ve"""),
																									        ((9, 1_5), tokenizer_r.mask_token),
																									        ((1_6, 2_1), """Allen"""),
																									        ((2_1, 2_3), """##NL"""),
																									        ((2_3, 2_4), """##P"""),
																									        ((2_5, 3_3), """sentence"""),
																									        ((3_3, 3_4), """."""),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									    if not do_lower_case
																									    else [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), """a"""),
																									        ((1, 2), ""","""),
																									        ((3, 8), """naive"""),
																									        ((9, 1_5), tokenizer_r.mask_token),
																									        ((1_6, 2_1), """allen"""),
																									        ((2_1, 2_3), """##nl"""),
																									        ((2_3, 2_4), """##p"""),
																									        ((2_5, 3_3), """sentence"""),
																									        ((3_3, 3_4), """."""),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									)

																									self.assertEqual(
																									    [e[1] for e in expected_results]      ,    tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]))
																									self.assertEqual([e[0] for e in expected_results]      ,    tokens["""offset_mapping"""])





				def 		_lowerCAmelCase		(      self							:							str):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						Optional[Any]			  =    ["""的""", """人""", """有"""]
											_SCREAMING_SNAKE_CASE		:						Any			  =    """""".join(_A)
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
																									_SCREAMING_SNAKE_CASE		:						Tuple			  =    True
																									_SCREAMING_SNAKE_CASE		:						str			  =    self.tokenizer_class.from_pretrained(_A      ,    **_A)
																									_SCREAMING_SNAKE_CASE		:						Dict			  =    self.rust_tokenizer_class.from_pretrained(_A      ,    **_A)

																									_SCREAMING_SNAKE_CASE		:						Dict			  =    tokenizer_p.encode(_A      ,    add_special_tokens=_A)
																									_SCREAMING_SNAKE_CASE		:						List[Any]			  =    tokenizer_r.encode(_A      ,    add_special_tokens=_A)

																									_SCREAMING_SNAKE_CASE		:						int			  =    tokenizer_r.convert_ids_to_tokens(_A)
																									_SCREAMING_SNAKE_CASE		:						List[str]			  =    tokenizer_p.convert_ids_to_tokens(_A)

																									# it is expected that each Chinese character is not preceded by "##"
																									self.assertListEqual(_A      ,    _A)
																									self.assertListEqual(_A      ,    _A)

																									_SCREAMING_SNAKE_CASE		:						Any			  =    False
																									_SCREAMING_SNAKE_CASE		:						Any			  =    self.rust_tokenizer_class.from_pretrained(_A      ,    **_A)
																									_SCREAMING_SNAKE_CASE		:						Optional[Any]			  =    self.tokenizer_class.from_pretrained(_A      ,    **_A)

																									_SCREAMING_SNAKE_CASE		:						Union[str, Any]			  =    tokenizer_r.encode(_A      ,    add_special_tokens=_A)
																									_SCREAMING_SNAKE_CASE		:						Optional[Any]			  =    tokenizer_p.encode(_A      ,    add_special_tokens=_A)

																									_SCREAMING_SNAKE_CASE		:						Union[str, Any]			  =    tokenizer_r.convert_ids_to_tokens(_A)
																									_SCREAMING_SNAKE_CASE		:						Optional[Any]			  =    tokenizer_p.convert_ids_to_tokens(_A)

																									# it is expected that only the first Chinese character is not preceded by "##".
																									_SCREAMING_SNAKE_CASE		:						Union[str, Any]			  =    [
																									    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_A)
																									]
																									self.assertListEqual(_A      ,    _A)
																									self.assertListEqual(_A      ,    _A)



"""simple docstring"""





import unittest

from transformers import is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_vision, slow, torch_device


if is_torch_available():
			import torch

			from transformers import AutoModelForImageClassification

if is_vision_available():
			from transformers import AutoImageProcessor




@require_torch
@require_vision
class  _snake_case		(    unittest.TestCase ):







				"""simple docstring"""






				@slow
				def 		_lowerCAmelCase		(      self							:							str):



											"""simple docstring"""







											_SCREAMING_SNAKE_CASE		:						Any			  =    AutoImageProcessor.from_pretrained("""microsoft/dit-base-finetuned-rvlcdip""")
											_SCREAMING_SNAKE_CASE		:						str			  =    AutoModelForImageClassification.from_pretrained("""microsoft/dit-base-finetuned-rvlcdip""")
											model.to(_A)

											from datasets import load_dataset

											_SCREAMING_SNAKE_CASE		:						Any			  =    load_dataset("""nielsr/rvlcdip-demo""")

											_SCREAMING_SNAKE_CASE		:						Any			  =    dataset["""train"""][0]["""image"""].convert("""RGB""")

											_SCREAMING_SNAKE_CASE		:						str			  =    image_processor(_A      ,    return_tensors="""pt""").to(_A)

											# forward pass
											with torch.no_grad():
																		_SCREAMING_SNAKE_CASE		:						Any			  =    model(**_A)
																		_SCREAMING_SNAKE_CASE		:						List[Any]			  =    outputs.logits

											_SCREAMING_SNAKE_CASE		:						List[str]			  =    torch.Size((1, 1_6))
											self.assertEqual(logits.shape      ,    _A)

											_SCREAMING_SNAKE_CASE		:						Optional[Any]			  =    torch.tensor(
											    [-0.4_158, -0.4_092, -0.4_347]      ,    device=_A      ,    dtype=torch.float      ,    )
											self.assertTrue(torch.allclose(logits[0, :3]      ,    _A      ,    atol=1e-4))


