






import collections
import os
import re
from pathlib import Path


lowerCamelCase : List[str]						   =       '''src/transformers'''


# Matches is_xxx_available()
lowerCamelCase : List[Any]						   =       re.compile(r'''is\_([a-z_]*)_available()''')
# Catches a one-line _import_struct = {xxx}
lowerCamelCase : Any						   =       re.compile(r'''^_import_structure\s+=\s+\{([^\}]+)\}''')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
lowerCamelCase : Union[str, Any]						   =       re.compile(r'''\s+"\S*":\s+\[([^\]]*)\]''')
# Catches a line if not is_foo_available
lowerCamelCase : Dict						   =       re.compile(r'''^\s*if\s+not\s+is\_[a-z_]*\_available\(\)''')
# Catches a line _import_struct["bla"].append("foo")
lowerCamelCase : Optional[Any]						   =       re.compile(r'''^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)''')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
lowerCamelCase : Any						   =       re.compile(r'''^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]''')
# Catches a line with an object between quotes and a comma:     "MyModel",
lowerCamelCase : str						   =       re.compile(r'''^\s+"([^"]+)",''')
# Catches a line with objects between brackets only:    ["foo", "bar"],
lowerCamelCase : Optional[Any]						   =       re.compile(r'''^\s+\[([^\]]+)\]''')
# Catches a line with from foo import bar, bla, boo
lowerCamelCase : Optional[int]						   =       re.compile(r'''\s+from\s+\S*\s+import\s+([^\(\s].*)\n''')
# Catches a line with try:
lowerCamelCase : str						   =       re.compile(r'''^\s*try:''')
# Catches a line with else:
lowerCamelCase : str						   =       re.compile(r'''^\s*else:''')
def       snake_case_		(     lowerCAmelCase_       :						Union[str, Any]      ):
							if _re_test_backend.search(lowerCAmelCase_      ) is None:
														return None
							__lowercase      :							str				=     [b[0] for b in _re_backend.findall(lowerCAmelCase_      )]
							backends.sort()
							return "_and_".join(lowerCAmelCase_      )
def       snake_case_		(     lowerCAmelCase_       :						Tuple      ):
							with open(lowerCAmelCase_       ,       """r"""       ,       encoding="""utf-8"""       ,       newline="""\n"""      ) as f:
														__lowercase      :							List[Any]				=     f.readlines()

							__lowercase      :							Optional[Any]				=     0
							while line_index < len(lowerCAmelCase_      ) and not lines[line_index].startswith("""_import_structure = {"""      ):
														line_index += 1

							# If this is a traditional init, just return.
							if line_index >= len(lowerCAmelCase_      ):
														return None

							# First grab the objects without a specific backend in _import_structure
							__lowercase      :							Dict				=     []
							while not lines[line_index].startswith("""if TYPE_CHECKING"""      ) and find_backend(lines[line_index]      ) is None:
														__lowercase      :							Tuple				=     lines[line_index]
														# If we have everything on a single line, let's deal with it.
														if _re_one_line_import_struct.search(lowerCAmelCase_      ):
																					__lowercase      :							Any				=     _re_one_line_import_struct.search(lowerCAmelCase_      ).groups()[0]
																					__lowercase      :							int				=     re.findall(r"""\[([^\]]+)\]"""       ,       lowerCAmelCase_      )
																					for imp in imports:
																												objects.extend([obj[1:-1] for obj in imp.split(""", """      )]      )
																					line_index += 1
																					continue
														__lowercase      :							List[str]				=     _re_import_struct_key_value.search(lowerCAmelCase_      )
														if single_line_import_search is not None:
																					__lowercase      :							int				=     [obj[1:-1] for obj in single_line_import_search.groups()[0].split(""", """      ) if len(lowerCAmelCase_      ) > 0]
																					objects.extend(lowerCAmelCase_      )
														elif line.startswith(""" """ * 8 + """\""""      ):
																					objects.append(line[9:-3]      )
														line_index += 1

							__lowercase      :							Tuple				=     {"""none""": objects}
							# Let's continue with backend-specific objects in _import_structure
							while not lines[line_index].startswith("""if TYPE_CHECKING"""      ):
														# If the line is an if not is_backend_available, we grab all objects associated.
														__lowercase      :							Optional[Any]				=     find_backend(lines[line_index]      )
														# Check if the backend declaration is inside a try block:
														if _re_try.search(lines[line_index - 1]      ) is None:
																					__lowercase      :							Dict				=     None

														if backend is not None:
																					line_index += 1

																					# Scroll until we hit the else block of try-except-else
																					while _re_else.search(lines[line_index]      ) is None:
																												line_index += 1

																					line_index += 1

																					__lowercase      :							Dict				=     []
																					# Until we unindent, add backend objects to the list
																					while len(lines[line_index]      ) <= 1 or lines[line_index].startswith(""" """ * 4      ):
																												__lowercase      :							Union[str, Any]				=     lines[line_index]
																												if _re_import_struct_add_one.search(lowerCAmelCase_      ) is not None:
																																			objects.append(_re_import_struct_add_one.search(lowerCAmelCase_      ).groups()[0]      )
																												elif _re_import_struct_add_many.search(lowerCAmelCase_      ) is not None:
																																			__lowercase      :							Optional[Any]				=     _re_import_struct_add_many.search(lowerCAmelCase_      ).groups()[0].split(""", """      )
																																			__lowercase      :							Optional[int]				=     [obj[1:-1] for obj in imports if len(lowerCAmelCase_      ) > 0]
																																			objects.extend(lowerCAmelCase_      )
																												elif _re_between_brackets.search(lowerCAmelCase_      ) is not None:
																																			__lowercase      :							Optional[int]				=     _re_between_brackets.search(lowerCAmelCase_      ).groups()[0].split(""", """      )
																																			__lowercase      :							int				=     [obj[1:-1] for obj in imports if len(lowerCAmelCase_      ) > 0]
																																			objects.extend(lowerCAmelCase_      )
																												elif _re_quote_object.search(lowerCAmelCase_      ) is not None:
																																			objects.append(_re_quote_object.search(lowerCAmelCase_      ).groups()[0]      )
																												elif line.startswith(""" """ * 8 + """\""""      ):
																																			objects.append(line[9:-3]      )
																												elif line.startswith(""" """ * 12 + """\""""      ):
																																			objects.append(line[13:-3]      )
																												line_index += 1

																					__lowercase      :							List[str]				=     objects
														else:
																					line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
							__lowercase      :							List[str]				=     []
							while (
							    line_index < len(lowerCAmelCase_      )
							    and find_backend(lines[line_index]      ) is None
							    and not lines[line_index].startswith("""else"""      )
							):
														__lowercase      :							Any				=     lines[line_index]
														__lowercase      :							Optional[Any]				=     _re_import.search(lowerCAmelCase_      )
														if single_line_import_search is not None:
																					objects.extend(single_line_import_search.groups()[0].split(""", """      )      )
														elif line.startswith(""" """ * 8      ):
																					objects.append(line[8:-2]      )
														line_index += 1

							__lowercase      :							Optional[int]				=     {"""none""": objects}
							# Let's continue with backend-specific objects
							while line_index < len(lowerCAmelCase_      ):
														# If the line is an if is_backend_available, we grab all objects associated.
														__lowercase      :							int				=     find_backend(lines[line_index]      )
														# Check if the backend declaration is inside a try block:
														if _re_try.search(lines[line_index - 1]      ) is None:
																					__lowercase      :							List[Any]				=     None

														if backend is not None:
																					line_index += 1

																					# Scroll until we hit the else block of try-except-else
																					while _re_else.search(lines[line_index]      ) is None:
																												line_index += 1

																					line_index += 1

																					__lowercase      :							str				=     []
																					# Until we unindent, add backend objects to the list
																					while len(lines[line_index]      ) <= 1 or lines[line_index].startswith(""" """ * 8      ):
																												__lowercase      :							List[Any]				=     lines[line_index]
																												__lowercase      :							List[Any]				=     _re_import.search(lowerCAmelCase_      )
																												if single_line_import_search is not None:
																																			objects.extend(single_line_import_search.groups()[0].split(""", """      )      )
																												elif line.startswith(""" """ * 12      ):
																																			objects.append(line[12:-2]      )
																												line_index += 1

																					__lowercase      :							List[str]				=     objects
														else:
																					line_index += 1

							return import_dict_objects, type_hint_objects
def       snake_case_		(     lowerCAmelCase_       :						Optional[int]       ,       lowerCAmelCase_       :						List[Any]      ):

							def find_duplicates(lowerCAmelCase_       :						Tuple      ):
														return [k for k, v in collections.Counter(lowerCAmelCase_      ).items() if v > 1]

							if list(import_dict_objects.keys()      ) != list(type_hint_objects.keys()      ):
														return ["Both sides of the init do not have the same backends!"]

							__lowercase      :							List[Any]				=     []
							for key in import_dict_objects.keys():
														__lowercase      :							Optional[Any]				=     find_duplicates(import_dict_objects[key]      )
														if duplicate_imports:
																					errors.append(F"Duplicate _import_structure definitions for: {duplicate_imports}"      )
														__lowercase      :							int				=     find_duplicates(type_hint_objects[key]      )
														if duplicate_type_hints:
																					errors.append(F"Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"      )

														if sorted(set(import_dict_objects[key]      )      ) != sorted(set(type_hint_objects[key]      )      ):
																					__lowercase      :							str				=     """base imports""" if key == """none""" else F"{key} backend"
																					errors.append(F"Differences for {name}:"      )
																					for a in type_hint_objects[key]:
																												if a not in import_dict_objects[key]:
																																			errors.append(F"  {a} in TYPE_HINT but not in _import_structure."      )
																					for a in import_dict_objects[key]:
																												if a not in type_hint_objects[key]:
																																			errors.append(F"  {a} in _import_structure but not in TYPE_HINT."      )
							return errors
def       snake_case_		(     ):
							__lowercase      :							List[str]				=     []
							for root, _, files in os.walk(lowerCAmelCase_      ):
														if "__init__.py" in files:
																					__lowercase      :							Any				=     os.path.join(lowerCAmelCase_       ,       """__init__.py"""      )
																					__lowercase      :							Optional[int]				=     parse_init(lowerCAmelCase_      )
																					if objects is not None:
																												__lowercase      :							List[str]				=     analyze_results(*lowerCAmelCase_      )
																												if len(lowerCAmelCase_      ) > 0:
																																			__lowercase      :							Optional[Any]				=     F"Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"
																																			failures.append("""\n""".join(lowerCAmelCase_      )      )
							if len(lowerCAmelCase_      ) > 0:
														raise ValueError("""\n\n""".join(lowerCAmelCase_      )      )
def       snake_case_		(     ):
							__lowercase      :							Tuple				=     []
							for path, directories, files in os.walk(lowerCAmelCase_      ):
														for folder in directories:
																					# Ignore private modules
																					if folder.startswith("""_"""      ):
																												directories.remove(lowerCAmelCase_      )
																												continue
																					# Ignore leftovers from branches (empty folders apart from pycache)
																					if len(list((Path(lowerCAmelCase_      ) / folder).glob("""*.py"""      )      )      ) == 0:
																												continue
																					__lowercase      :							Optional[Any]				=     str((Path(lowerCAmelCase_      ) / folder).relative_to(lowerCAmelCase_      )      )
																					__lowercase      :							Optional[Any]				=     short_path.replace(os.path.sep       ,       """."""      )
																					submodules.append(lowerCAmelCase_      )
														for fname in files:
																					if fname == "__init__.py":
																												continue
																					__lowercase      :							Optional[int]				=     str((Path(lowerCAmelCase_      ) / fname).relative_to(lowerCAmelCase_      )      )
																					__lowercase      :							List[str]				=     short_path.replace(""".py"""       ,       """"""      ).replace(os.path.sep       ,       """."""      )
																					if len(submodule.split("""."""      )      ) == 1:
																												submodules.append(lowerCAmelCase_      )
							return submodules


lowerCamelCase : Optional[int]						   =       [
    '''convert_pytorch_checkpoint_to_tf2''',
    '''modeling_flax_pytorch_utils''',
    '''models.esm.openfold_utils''',
]
def       snake_case_		(     ):
							# This is to make sure the transformers module imported is the one in the repo.
							from transformers.utils import direct_transformers_import

							__lowercase      :							List[str]				=     direct_transformers_import(lowerCAmelCase_      )

							__lowercase      :							Tuple				=     set(transformers._import_structure.keys()      )
							# This contains all the base keys of the _import_structure object defined in the init, but if the user is missing
							# some optional dependencies, they may not have all of them. Thus we read the init to read all additions and
							# (potentiall re-) add them.
							with open(os.path.join(lowerCAmelCase_       ,       """__init__.py"""      )       ,       """r"""      ) as f:
														__lowercase      :							List[str]				=     f.read()
							import_structure_keys.update(set(re.findall(r"""import_structure\[\"([^\"]*)\"\]"""       ,       lowerCAmelCase_      )      )      )

							__lowercase      :							List[Any]				=     [
							    module
							    for module in get_transformers_submodules()
							    if module not in IGNORE_SUBMODULES and module not in import_structure_keys
							]

							if len(lowerCAmelCase_      ) > 0:
														__lowercase      :							List[Any]				=     """\n""".join(F"- {module}" for module in module_not_registered      )
														raise ValueError(
														    """The following submodules are not properly registed in the main init of Transformers:\n"""
														    F"{list_of_modules}\n"
														    """Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."""      )


if __name__ == "__main__":
	check_all_inits()
	check_submodules()







import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		lowerCAmelCase     ( __a      ,       unittest.TestCase			):

					'''simple docstring'''

					_A						:			str									=      LongformerTokenizer
					_A						:			int									=      True
					_A						:			Optional[int]									=      LongformerTokenizerFast
					_A						:			int									=      True



					def       lowerCAmelCase				(  self				:	Any							)			->  List[str]:


												"""simple docstring"""

												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												__lowercase      :							Union[str, Any]				=     [
												    """l""",
												    """o""",
												    """w""",
												    """e""",
												    """r""",
												    """s""",
												    """t""",
												    """i""",
												    """d""",
												    """n""",
												    """\u0120""",
												    """\u0120l""",
												    """\u0120n""",
												    """\u0120lo""",
												    """\u0120low""",
												    """er""",
												    """\u0120lowest""",
												    """\u0120newer""",
												    """\u0120wider""",
												    """<unk>""",
												]
												__lowercase      :							Union[str, Any]				=     dict(zip(__a		,     range(len(__a							)							)							)							)
												__lowercase      :							Any				=     ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
												__lowercase      :							Optional[int]				=     {"""unk_token""": """<unk>"""}

												__lowercase      :							Dict				=     os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["""vocab_file"""]							)
												__lowercase      :							str				=     os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["""merges_file"""]							)
												with open(self.vocab_file		,     """w"""		,     encoding="""utf-8"""							) as fp:
																			fp.write(json.dumps(__a							) + """\n"""							)
												with open(self.merges_file		,     """w"""		,     encoding="""utf-8"""							) as fp:
																			fp.write("""\n""".join(__a							)							)



					def       lowerCAmelCase				(  self				:	Optional[int]		,     **__a				:	Optional[Any]							)			->  str:


												"""simple docstring"""

												kwargs.update(self.special_tokens_map							)
												return self.tokenizer_class.from_pretrained(self.tmpdirname		,     **__a							)



					def       lowerCAmelCase				(  self				:	Tuple		,     **__a				:	Tuple							)			->  str:


												"""simple docstring"""

												kwargs.update(self.special_tokens_map							)
												return self.rust_tokenizer_class.from_pretrained(self.tmpdirname		,     **__a							)



					def       lowerCAmelCase				(  self				:	str		,     __a				:	Optional[int]							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     """lower newer"""
												__lowercase      :							int				=     """lower newer"""
												return input_text, output_text



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     self.tokenizer_class(self.vocab_file		,     self.merges_file		,     **self.special_tokens_map							)
												__lowercase      :							Dict				=     """lower newer"""
												__lowercase      :							Optional[Any]				=     ["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
												__lowercase      :							str				=     tokenizer.tokenize(__a							)  # , add_prefix_space=True)
												self.assertListEqual(__a		,     __a							)

												__lowercase      :							int				=     tokens + [tokenizer.unk_token]
												__lowercase      :							str				=     [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(__a							)		,     __a							)



					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     self.get_tokenizer()

												self.assertListEqual(tokenizer.encode("""Hello world!"""		,     add_special_tokens=__a							)		,     [0, 31414, 232, 328, 2]							)
												self.assertListEqual(
												    tokenizer.encode("""Hello world! cécé herlolip 418"""		,     add_special_tokens=__a							)		,     [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]		,     )



					@slow
					def       lowerCAmelCase				(  self				:	Tuple							)			->  str:


												"""simple docstring"""

												__lowercase      :							Any				=     self.tokenizer_class.from_pretrained("""allenai/longformer-base-4096"""							)

												__lowercase      :							Optional[Any]				=     tokenizer.encode("""sequence builders"""		,     add_special_tokens=__a							)
												__lowercase      :							List[str]				=     tokenizer.encode("""multi-sequence build"""		,     add_special_tokens=__a							)

												__lowercase      :							Optional[Any]				=     tokenizer.encode(
												    """sequence builders"""		,     add_special_tokens=__a		,     add_prefix_space=__a							)
												__lowercase      :							Union[str, Any]				=     tokenizer.encode(
												    """sequence builders"""		,     """multi-sequence build"""		,     add_special_tokens=__a		,     add_prefix_space=__a							)

												__lowercase      :							List[Any]				=     tokenizer.build_inputs_with_special_tokens(__a							)
												__lowercase      :							Any				=     tokenizer.build_inputs_with_special_tokens(__a		,     __a							)

												assert encoded_sentence == encoded_text_from_decode
												assert encoded_pair == encoded_pair_from_decode



					def       lowerCAmelCase				(  self				:	Dict							)			->  Tuple:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     self.get_tokenizer()

												__lowercase      :							Tuple				=     """Encode this sequence."""
												__lowercase      :							Optional[Any]				=     tokenizer.byte_encoder[""" """.encode("""utf-8"""							)[0]]

												# Testing encoder arguments
												__lowercase      :							Dict				=     tokenizer.encode(__a		,     add_special_tokens=__a		,     add_prefix_space=__a							)
												__lowercase      :							Tuple				=     tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
												self.assertNotEqual(__a		,     __a							)

												__lowercase      :							List[str]				=     tokenizer.encode(__a		,     add_special_tokens=__a		,     add_prefix_space=__a							)
												__lowercase      :							Any				=     tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
												self.assertEqual(__a		,     __a							)

												tokenizer.add_special_tokens({"""bos_token""": """<s>"""}							)
												__lowercase      :							str				=     tokenizer.encode(__a		,     add_special_tokens=__a							)
												__lowercase      :							Dict				=     tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
												self.assertNotEqual(__a		,     __a							)

												# Testing spaces after special tokens
												__lowercase      :							List[Any]				=     """<mask>"""
												tokenizer.add_special_tokens(
												    {"""mask_token""": AddedToken(__a		,     lstrip=__a		,     rstrip=__a							)}							)  # mask token has a left space
												__lowercase      :							Dict				=     tokenizer.convert_tokens_to_ids(__a							)

												__lowercase      :							List[str]				=     """Encode <mask> sequence"""
												__lowercase      :							List[str]				=     """Encode <mask>sequence"""

												__lowercase      :							Union[str, Any]				=     tokenizer.encode(__a							)
												__lowercase      :							Dict				=     encoded.index(__a							)
												__lowercase      :							List[Any]				=     tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
												self.assertEqual(__a		,     __a							)

												__lowercase      :							int				=     tokenizer.encode(__a							)
												__lowercase      :							Union[str, Any]				=     encoded.index(__a							)
												__lowercase      :							List[Any]				=     tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
												self.assertNotEqual(__a		,     __a							)



					def       lowerCAmelCase				(  self				:	int							)			->  str:


												"""simple docstring"""

												pass



					def       lowerCAmelCase				(  self				:	int							)			->  Dict:


												"""simple docstring"""

												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
																										__lowercase      :							List[str]				=     self.rust_tokenizer_class.from_pretrained(__a		,     **__a							)
																										__lowercase      :							List[Any]				=     self.tokenizer_class.from_pretrained(__a		,     **__a							)
																										__lowercase      :							Optional[Any]				=     """A, <mask> AllenNLP sentence."""
																										__lowercase      :							Union[str, Any]				=     tokenizer_r.encode_plus(__a		,     add_special_tokens=__a		,     return_token_type_ids=__a							)
																										__lowercase      :							Optional[Any]				=     tokenizer_p.encode_plus(__a		,     add_special_tokens=__a		,     return_token_type_ids=__a							)

																										# token_type_ids should put 0 everywhere
																										self.assertEqual(sum(tokens_r["""token_type_ids"""]							)		,     sum(tokens_p["""token_type_ids"""]							)							)

																										# attention_mask should put 1 everywhere, so sum over length should be 1
																										self.assertEqual(
																										    sum(tokens_r["""attention_mask"""]							) / len(tokens_r["""attention_mask"""]							)		,     sum(tokens_p["""attention_mask"""]							) / len(tokens_p["""attention_mask"""]							)		,     )

																										__lowercase      :							Dict				=     tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]							)
																										__lowercase      :							str				=     tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]							)

																										# Rust correctly handles the space before the mask while python doesnt
																										self.assertSequenceEqual(tokens_p["""input_ids"""]		,     [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]							)
																										self.assertSequenceEqual(tokens_r["""input_ids"""]		,     [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]							)

																										self.assertSequenceEqual(
																										    __a		,     ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)
																										self.assertSequenceEqual(
																										    __a		,     ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Dict:


												"""simple docstring"""

												for trim_offsets, add_prefix_space in itertools.product([True, False]		,     repeat=2							):
																			__lowercase      :							Dict				=     self.rust_tokenizer_class.from_pretrained(
																			    self.tmpdirname		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)

																			__lowercase      :							Optional[int]				=     json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
																			__lowercase      :							Any				=     json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

																			self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]		,     __a							)

																			self.assertEqual(post_processor_state["""add_prefix_space"""]		,     __a							)
																			self.assertEqual(post_processor_state["""trim_offsets"""]		,     __a							)



					def       lowerCAmelCase				(  self				:	int							)			->  Tuple:


												"""simple docstring"""

												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
																										__lowercase      :							List[str]				=     """hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
																										__lowercase      :							int				=     F"{text_of_1_token} {text_of_1_token}"

																										__lowercase      :							List[str]				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Any				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							) + 1, len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							str				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Tuple				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							) + 1, len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							Optional[int]				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							str				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							), len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							str				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							int				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(__a							), len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							Any				=     F" {text}"

																										# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																										#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																										# )
																										# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																										# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																										# self.assertEqual(
																										#     encoding.offset_mapping[1],
																										#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																										# )

																										__lowercase      :							Optional[Any]				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							str				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (1, 1 + len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(__a							) + 1, 1 + len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							int				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Dict				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, 1 + len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(__a							), 1 + len(__a							) + 1 + len(__a							))		,     )

																										__lowercase      :							int				=     self.rust_tokenizer_class.from_pretrained(
																										    __a		,     use_fast=__a		,     add_prefix_space=__a		,     trim_offsets=__a							)
																										__lowercase      :							Tuple				=     tokenizer_r(__a		,     return_offsets_mapping=__a		,     add_special_tokens=__a							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, 1 + len(__a							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(__a							), 1 + len(__a							) + 1 + len(__a							))		,     )