



import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class   _A   ( _lowercase  ,						unittest.TestCase	):


						'''simple docstring'''





						_snake_case			:     str      							=      MobileBertTokenizer
						_snake_case			:     Dict      							=      MobileBertTokenizerFast
						_snake_case			:     Optional[int]      							=      True
						_snake_case			:     str      							=      True
						_snake_case			:     Union[str, Any]      							=      filter_non_english
						_snake_case			:     Optional[int]      							=      """google/mobilebert-uncased"""



						def     _snake_case	(	self	:							Tuple	):






							'''simple docstring'''



							super().setUp()

							__lowercase		       =			[
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							__lowercase		       =			os.path.join(self.tmpdirname							,      VOCAB_FILES_NAMES["vocab_file"]	)
							with open(self.vocab_file							,      "w"							,      encoding="utf-8"	) as vocab_writer:
								vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)

							__lowercase		       =			[
							    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
							    for tokenizer_def in self.tokenizers_list
							]



						def     _snake_case	(	self	:							Union[str, Any]							,      lowerCamelCase	:							Any	):






							'''simple docstring'''



							__lowercase		       =			"UNwant\u00E9d,running"
							__lowercase		       =			"unwanted, running"
							return input_text, output_text



						def     _snake_case	(	self	:							List[str]	):






							'''simple docstring'''



							__lowercase		       =			self.tokenizer_class(self.vocab_file	)

							__lowercase		       =			tokenizer.tokenize("UNwant\u00E9d,running"	)
							self.assertListEqual(lowerCamelCase							,      ["un", "##want", "##ed", ",", "runn", "##ing"]	)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase	)							,      [9, 6, 7, 12, 10, 11]	)



						def     _snake_case	(	self	:							List[str]	):






							'''simple docstring'''



							if not self.test_rust_tokenizer:
								return

							__lowercase		       =			self.get_tokenizer()
							__lowercase		       =			self.get_rust_tokenizer()

							__lowercase		       =			"UNwant\u00E9d,running"

							__lowercase		       =			tokenizer.tokenize(lowerCamelCase	)
							__lowercase		       =			rust_tokenizer.tokenize(lowerCamelCase	)
							self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)

							__lowercase		       =			tokenizer.encode(lowerCamelCase							,      add_special_tokens=lowerCamelCase	)
							__lowercase		       =			rust_tokenizer.encode(lowerCamelCase							,      add_special_tokens=lowerCamelCase	)
							self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)

							__lowercase		       =			self.get_rust_tokenizer()
							__lowercase		       =			tokenizer.encode(lowerCamelCase	)
							__lowercase		       =			rust_tokenizer.encode(lowerCamelCase	)
							self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)

							# With lower casing
							__lowercase		       =			self.get_tokenizer(do_lower_case=lowerCamelCase	)
							__lowercase		       =			self.get_rust_tokenizer(do_lower_case=lowerCamelCase	)

							__lowercase		       =			"UNwant\u00E9d,running"

							__lowercase		       =			tokenizer.tokenize(lowerCamelCase	)
							__lowercase		       =			rust_tokenizer.tokenize(lowerCamelCase	)
							self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)

							__lowercase		       =			tokenizer.encode(lowerCamelCase							,      add_special_tokens=lowerCamelCase	)
							__lowercase		       =			rust_tokenizer.encode(lowerCamelCase							,      add_special_tokens=lowerCamelCase	)
							self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)

							__lowercase		       =			self.get_rust_tokenizer()
							__lowercase		       =			tokenizer.encode(lowerCamelCase	)
							__lowercase		       =			rust_tokenizer.encode(lowerCamelCase	)
							self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)



						def     _snake_case	(	self	:							int	):






							'''simple docstring'''



							__lowercase		       =			BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)							,      ["ah", "\u535A", "\u63A8", "zz"]	)



						def     _snake_case	(	self	:							List[Any]	):






							'''simple docstring'''



							__lowercase		       =			BasicTokenizer(do_lower_case=lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)							,      ["hello", "!", "how", "are", "you", "?"]	)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)							,      ["hello"]	)



						def     _snake_case	(	self	:							Any	):






							'''simple docstring'''



							__lowercase		       =			BasicTokenizer(do_lower_case=lowerCamelCase							,      strip_accents=lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)							,      ["hällo", "!", "how", "are", "you", "?"]	)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)							,      ["h\u00E9llo"]	)



						def     _snake_case	(	self	:							Optional[int]	):






							'''simple docstring'''



							__lowercase		       =			BasicTokenizer(do_lower_case=lowerCamelCase							,      strip_accents=lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)							,      ["hallo", "!", "how", "are", "you", "?"]	)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)							,      ["hello"]	)



						def     _snake_case	(	self	:							Optional[Any]	):






							'''simple docstring'''



							__lowercase		       =			BasicTokenizer(do_lower_case=lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)							,      ["hallo", "!", "how", "are", "you", "?"]	)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)							,      ["hello"]	)



						def     _snake_case	(	self	:							int	):






							'''simple docstring'''



							__lowercase		       =			BasicTokenizer(do_lower_case=lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)							,      ["HeLLo", "!", "how", "Are", "yoU", "?"]	)



						def     _snake_case	(	self	:							List[str]	):






							'''simple docstring'''



							__lowercase		       =			BasicTokenizer(do_lower_case=lowerCamelCase							,      strip_accents=lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)							,      ["HäLLo", "!", "how", "Are", "yoU", "?"]	)



						def     _snake_case	(	self	:							Tuple	):






							'''simple docstring'''



							__lowercase		       =			BasicTokenizer(do_lower_case=lowerCamelCase							,      strip_accents=lowerCamelCase	)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)							,      ["HaLLo", "!", "how", "Are", "yoU", "?"]	)



						def     _snake_case	(	self	:							List[str]	):






							'''simple docstring'''



							__lowercase		       =			BasicTokenizer(do_lower_case=lowerCamelCase							,      never_split=["[UNK]"]	)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)							,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)



						def     _snake_case	(	self	:							int	):






							'''simple docstring'''



							__lowercase		       =			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							__lowercase		       =			{}
							for i, token in enumerate(lowerCamelCase	):
								__lowercase		       =			i
							__lowercase		       =			WordpieceTokenizer(vocab=lowerCamelCase							,      unk_token="[UNK]"	)

							self.assertListEqual(tokenizer.tokenize(""	)							,      []	)

							self.assertListEqual(tokenizer.tokenize("unwanted running"	)							,      ["un", "##want", "##ed", "runn", "##ing"]	)

							self.assertListEqual(tokenizer.tokenize("unwantedX running"	)							,      ["[UNK]", "runn", "##ing"]	)



						def     _snake_case	(	self	:							Optional[Any]	):






							'''simple docstring'''



							self.assertTrue(_is_whitespace(" "	)	)
							self.assertTrue(_is_whitespace("\t"	)	)
							self.assertTrue(_is_whitespace("\r"	)	)
							self.assertTrue(_is_whitespace("\n"	)	)
							self.assertTrue(_is_whitespace("\u00A0"	)	)

							self.assertFalse(_is_whitespace("A"	)	)
							self.assertFalse(_is_whitespace("-"	)	)



						def     _snake_case	(	self	:							List[str]	):






							'''simple docstring'''



							self.assertTrue(_is_control("\u0005"	)	)

							self.assertFalse(_is_control("A"	)	)
							self.assertFalse(_is_control(" "	)	)
							self.assertFalse(_is_control("\t"	)	)
							self.assertFalse(_is_control("\r"	)	)



						def     _snake_case	(	self	:							Any	):






							'''simple docstring'''



							self.assertTrue(_is_punctuation("-"	)	)
							self.assertTrue(_is_punctuation("$"	)	)
							self.assertTrue(_is_punctuation("`"	)	)
							self.assertTrue(_is_punctuation("."	)	)

							self.assertFalse(_is_punctuation("A"	)	)
							self.assertFalse(_is_punctuation(" "	)	)



						def     _snake_case	(	self	:							List[str]	):






							'''simple docstring'''



							__lowercase		       =			self.get_tokenizer()
							__lowercase		       =			self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(lowerCamelCase	) for t in ["Test", "\xad", "test"]]							,      [["[UNK]"], [], ["[UNK]"]]	)

							self.assertListEqual(
							    [rust_tokenizer.tokenize(lowerCamelCase	) for t in ["Test", "\xad", "test"]]							,      [["[UNK]"], [], ["[UNK]"]]	)



						@slow
						def     _snake_case	(	self	:							str	):






							'''simple docstring'''



							__lowercase		       =			self.tokenizer_class.from_pretrained("google/mobilebert-uncased"	)

							__lowercase		       =			tokenizer.encode("sequence builders"							,      add_special_tokens=lowerCamelCase	)
							__lowercase		       =			tokenizer.encode("multi-sequence build"							,      add_special_tokens=lowerCamelCase	)

							__lowercase		       =			tokenizer.build_inputs_with_special_tokens(lowerCamelCase	)
							__lowercase		       =			tokenizer.build_inputs_with_special_tokens(lowerCamelCase							,      lowerCamelCase	)

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]



						def     _snake_case	(	self	:							Tuple	):






							'''simple docstring'''



							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
									__lowercase		       =			self.rust_tokenizer_class.from_pretrained(lowerCamelCase							,      **lowerCamelCase	)

									__lowercase		       =			f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
									__lowercase		       =			tokenizer_r.encode_plus(
									    lowerCamelCase							,      return_attention_mask=lowerCamelCase							,      return_token_type_ids=lowerCamelCase							,      return_offsets_mapping=lowerCamelCase							,      add_special_tokens=lowerCamelCase							,      )

									__lowercase		       =			tokenizer_r.do_lower_case if hasattr(lowerCamelCase							,      "do_lower_case"	) else False
									__lowercase		       =			(
									    [
									        ((0, 0), tokenizer_r.cls_token),
									        ((0, 1), "A"),
									        ((1, 2), ","),
									        ((3, 5), "na"),
									        ((5, 6), "##ï"),
									        ((6, 8), "##ve"),
									        ((9, 15), tokenizer_r.mask_token),
									        ((16, 21), "Allen"),
									        ((21, 23), "##NL"),
									        ((23, 24), "##P"),
									        ((25, 33), "sentence"),
									        ((33, 34), "."),
									        ((0, 0), tokenizer_r.sep_token),
									    ]
									    if not do_lower_case
									    else [
									        ((0, 0), tokenizer_r.cls_token),
									        ((0, 1), "a"),
									        ((1, 2), ","),
									        ((3, 8), "naive"),
									        ((9, 15), tokenizer_r.mask_token),
									        ((16, 21), "allen"),
									        ((21, 23), "##nl"),
									        ((23, 24), "##p"),
									        ((25, 33), "sentence"),
									        ((33, 34), "."),
									        ((0, 0), tokenizer_r.sep_token),
									    ]
									)

									self.assertEqual(
									    [e[1] for e in expected_results]							,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
									self.assertEqual([e[0] for e in expected_results]							,      tokens["offset_mapping"]	)



						def     _snake_case	(	self	:							List[str]	):






							'''simple docstring'''



							__lowercase		       =			["的", "人", "有"]
							__lowercase		       =			"".join(lowerCamelCase	)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
									__lowercase		       =			True
									__lowercase		       =			self.tokenizer_class.from_pretrained(lowerCamelCase							,      **lowerCamelCase	)
									__lowercase		       =			self.rust_tokenizer_class.from_pretrained(lowerCamelCase							,      **lowerCamelCase	)

									__lowercase		       =			tokenizer_p.encode(lowerCamelCase							,      add_special_tokens=lowerCamelCase	)
									__lowercase		       =			tokenizer_r.encode(lowerCamelCase							,      add_special_tokens=lowerCamelCase	)

									__lowercase		       =			tokenizer_r.convert_ids_to_tokens(lowerCamelCase	)
									__lowercase		       =			tokenizer_p.convert_ids_to_tokens(lowerCamelCase	)

									# it is expected that each Chinese character is not preceded by "##"
									self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)
									self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)

									__lowercase		       =			False
									__lowercase		       =			self.rust_tokenizer_class.from_pretrained(lowerCamelCase							,      **lowerCamelCase	)
									__lowercase		       =			self.tokenizer_class.from_pretrained(lowerCamelCase							,      **lowerCamelCase	)

									__lowercase		       =			tokenizer_r.encode(lowerCamelCase							,      add_special_tokens=lowerCamelCase	)
									__lowercase		       =			tokenizer_p.encode(lowerCamelCase							,      add_special_tokens=lowerCamelCase	)

									__lowercase		       =			tokenizer_r.convert_ids_to_tokens(lowerCamelCase	)
									__lowercase		       =			tokenizer_p.convert_ids_to_tokens(lowerCamelCase	)

									# it is expected that only the first Chinese character is not preceded by "##".
									__lowercase		       =			[
									    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase	)
									]
									self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)
									self.assertListEqual(lowerCamelCase							,      lowerCamelCase	)








from __future__ import annotations

from collections.abc import Callable
from typing import Generic, TypeVar

snake_case__					:     Union[str, Any]    		=   TypeVar("""T""")
snake_case__					:     Optional[int]    		=   TypeVar("""U""")

class   _A   ( Generic[T, U]	):


						'''simple docstring'''





						def __init__(	self	:							Optional[int]							,      lowerCamelCase	:							T | None							,      lowerCamelCase	:							U | None	):






							'''simple docstring'''



							__lowercase		       =			key
							__lowercase		       =			val
							__lowercase		       =			None
							__lowercase		       =			None



						def __repr__(	self	:							Any	):






							'''simple docstring'''



							return (
							    f"""Node: key: {self.key}, val: {self.val}, """
							    f"""has next: {bool(self.next	)}, has prev: {bool(self.prev	)}"""
							)

class   _A   ( Generic[T, U]	):


						'''simple docstring'''





						def __init__(	self	:							Dict	):






							'''simple docstring'''



							__lowercase		       =			DoubleLinkedListNode(lowerCamelCase							,      lowerCamelCase	)
							__lowercase		       =			DoubleLinkedListNode(lowerCamelCase							,      lowerCamelCase	)
							__lowercase       ,    __lowercase		       =			self.rear, self.head



						def __repr__(	self	:							Optional[Any]	):






							'''simple docstring'''



							__lowercase		       =			["DoubleLinkedList"]
							__lowercase		       =			self.head
							while node.next is not None:
								rep.append(str(lowerCamelCase	)	)
								__lowercase		       =			node.next
							rep.append(str(self.rear	)	)
							return ",\n    ".join(lowerCamelCase	)



						def     _snake_case	(	self	:							Union[str, Any]							,      lowerCamelCase	:							DoubleLinkedListNode[T, U]	):






							'''simple docstring'''



							__lowercase		       =			self.rear.prev

							# All nodes other than self.head are guaranteed to have non-None previous
							assert previous is not None

							__lowercase		       =			node
							__lowercase		       =			previous
							__lowercase		       =			node
							__lowercase		       =			self.rear



						def     _snake_case	(	self	:							Optional[int]							,      lowerCamelCase	:							DoubleLinkedListNode[T, U]	):






							'''simple docstring'''



							if node.prev is None or node.next is None:
								return None

							__lowercase		       =			node.next
							__lowercase		       =			node.prev
							__lowercase		       =			None
							__lowercase		       =			None
							return node

class   _A   ( Generic[T, U]	):


						'''simple docstring'''





						_snake_case			:     dict[Callable[[T], U], LRUCache[T, U]]      							=      {}



						def __init__(	self	:							List[Any]							,      lowerCamelCase	:							int	):






							'''simple docstring'''



							__lowercase		       =			DoubleLinkedList()
							__lowercase		       =			capacity
							__lowercase		       =			0
							__lowercase		       =			0
							__lowercase		       =			0
							__lowercase		       =			{}



						def __repr__(	self	:							Optional[Any]	):






							'''simple docstring'''



							return (
							    f"""CacheInfo(hits={self.hits}, misses={self.miss}, """
							    f"""capacity={self.capacity}, current size={self.num_keys})"""
							)



						def __contains__(	self	:							Dict							,      lowerCamelCase	:							T	):






							'''simple docstring'''



							return key in self.cache



						def     _snake_case	(	self	:							List[Any]							,      lowerCamelCase	:							T	):






							'''simple docstring'''



							if key in self.cache:
								self.hits += 1
								__lowercase		       =			self.cache[key]
								__lowercase		       =			self.list.remove(self.cache[key]	)
								assert node == value_node

								# node is guaranteed not None because it is in self.cache
								assert node is not None
								self.list.add(lowerCamelCase	)
								return node.val
							self.miss += 1
							return None



						def     _snake_case	(	self	:							Union[str, Any]							,      lowerCamelCase	:							T							,      lowerCamelCase	:							U	):






							'''simple docstring'''



							if key not in self.cache:
								if self.num_keys >= self.capacity:
									# delete first node (oldest) when over capacity
									__lowercase		       =			self.list.head.next

									# guaranteed to have a non-None first node when num_keys > 0
									# explain to type checker via assertions
									assert first_node is not None
									assert first_node.key is not None
									assert (
									    self.list.remove(lowerCamelCase	) is not None
									)  # node guaranteed to be in list assert node.key is not None

									del self.cache[first_node.key]
									self.num_keys -= 1
								__lowercase		       =			DoubleLinkedListNode(lowerCamelCase							,      lowerCamelCase	)
								self.list.add(self.cache[key]	)
								self.num_keys += 1

							else:
								# bump node to the end of the list, update value
								__lowercase		       =			self.list.remove(self.cache[key]	)
								assert node is not None  # node guaranteed to be in list
								__lowercase		       =			value
								self.list.add(lowerCamelCase	)



						@classmethod
						def     _snake_case	(	cls	:							Union[str, Any]							,      lowerCamelCase	:							int = 128	):






							'''simple docstring'''



							def cache_decorator_inner(lowerCamelCase	:							Callable[[T], U]	) -> Callable[..., U]:
								def cache_decorator_wrapper(*lowerCamelCase	:							T	) -> U:
									if func not in cls.decorator_function_to_instance_map:
										__lowercase		       =			LRUCache(lowerCamelCase	)

									__lowercase		       =			cls.decorator_function_to_instance_map[func].get(args[0]	)
									if result is None:
										__lowercase		       =			func(*lowerCamelCase	)
										cls.decorator_function_to_instance_map[func].put(args[0]							,      lowerCamelCase	)
									return result

								def cache_info() -> LRUCache[T, U]:
									return cls.decorator_function_to_instance_map[func]

								setattr(lowerCamelCase							,      "cache_info"							,      lowerCamelCase	)  # noqa: B010

								return cache_decorator_wrapper

							return cache_decorator_inner


if __name__ == "__main__":
	import doctest

	doctest.testmod()



