


import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
							a					  =     'pt'
elif is_tf_available():
							a					  =     'tf'
else:
							a					  =     'jax'







class        UpperCamelCase__   (	__magic_name__							,				unittest.TestCase   ):
					__SCREAMING_SNAKE_CASE       :  Optional[int]						=   ByTaTokenizer
					__SCREAMING_SNAKE_CASE       :  List[str]						=   False

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										super().setUp()
										lowercase_  							=			ByTaTokenizer()
										tokenizer.save_pretrained(self.tmpdirname       )

					@cached_property
					def   UpperCAmelCase__       (     self       :   Any       ):






										'''simple docstring'''






										return ByTaTokenizer.from_pretrained("""google/byt5-small"""       )

					def   UpperCAmelCase__       (     self       :   Union[str, Any]							,					**UpperCamelCase__       :   Union[str, Any]       ):






										'''simple docstring'''






										return self.tokenizer_class.from_pretrained(self.tmpdirname							,					**UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   Any							,					UpperCamelCase__       :   Tuple							,					UpperCamelCase__       :   Optional[Any]=False							,					UpperCamelCase__       :   Optional[int]=20							,					UpperCamelCase__       :   List[str]=5       ):






										'''simple docstring'''






										lowercase_  							=			[]
										for i in range(len(UpperCamelCase__       )       ):
															try:
																				lowercase_  							=			tokenizer.decode([i]							,					clean_up_tokenization_spaces=UpperCamelCase__       )
															except UnicodeDecodeError:
																				pass
															toks.append((i, tok)       )

										lowercase_  							=			list(filter(lambda UpperCamelCase__       : re.match(R"""^[ a-zA-Z]+$"""							,					t[1]       )							,					UpperCamelCase__       )       )
										lowercase_  							=			list(filter(lambda UpperCamelCase__       : [t[0]] == tokenizer.encode(t[1]							,					add_special_tokens=UpperCamelCase__       )							,					UpperCamelCase__       )       )
										if max_length is not None and len(UpperCamelCase__       ) > max_length:
															lowercase_  							=			toks[:max_length]
										if min_length is not None and len(UpperCamelCase__       ) < min_length and len(UpperCamelCase__       ) > 0:
															while len(UpperCamelCase__       ) < min_length:
																				lowercase_  							=			toks + toks
        # toks_str = [t[1] for t in toks]
										lowercase_  							=			[t[0] for t in toks]

										# Ensure consistency
										lowercase_  							=			tokenizer.decode(UpperCamelCase__							,					clean_up_tokenization_spaces=UpperCamelCase__       )
										if " " not in output_txt and len(UpperCamelCase__       ) > 1:
															lowercase_  							=			(
															    tokenizer.decode([toks_ids[0]]							,					clean_up_tokenization_spaces=UpperCamelCase__       )
															    + """ """
															    + tokenizer.decode(toks_ids[1:]							,					clean_up_tokenization_spaces=UpperCamelCase__       )
															)
										if with_prefix_space:
															lowercase_  							=			""" """ + output_txt
										lowercase_  							=			tokenizer.encode(UpperCamelCase__							,					add_special_tokens=UpperCamelCase__       )
										return output_txt, output_ids

					def   UpperCAmelCase__       (     self       :   Union[str, Any]       ):






										'''simple docstring'''






										lowercase_  							=			self.ta_base_tokenizer
										lowercase_  							=			tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]       )
										lowercase_  							=			tokenizer(["""hi""", """I went to the gym""", """"""]       )
										self.assertListEqual(batch_with_eos_added["""input_ids"""]							,					batch_without_eos_added["""input_ids"""]       )

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										lowercase_  							=			self.ta_base_tokenizer
										lowercase_  							=			"""Unicode €."""
										lowercase_  							=			tokenizer(UpperCamelCase__       )
										lowercase_  							=			[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
										self.assertEqual(encoded["""input_ids"""]							,					UpperCamelCase__       )

										# decoding
										lowercase_  							=			tokenizer.decode(UpperCamelCase__       )
										self.assertEqual(UpperCamelCase__							,					"""Unicode €.</s>"""       )

										lowercase_  							=			tokenizer("""e è é ê ë"""       )
										lowercase_  							=			[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
										self.assertEqual(encoded["""input_ids"""]							,					UpperCamelCase__       )
										# decoding
										lowercase_  							=			tokenizer.decode(UpperCamelCase__       )
										self.assertEqual(UpperCamelCase__							,					"""e è é ê ë</s>"""       )

										# encode/decode, but with `encode` instead of `__call__`
										self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""       )       )							,					"""e è é ê ë</s>"""       )

					def   UpperCAmelCase__       (     self       :   Optional[Any]       ):






										'''simple docstring'''






										lowercase_  							=			self.ta_base_tokenizer
										lowercase_  							=			["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
										# fmt: off
										lowercase_  							=			[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
										# fmt: on
										lowercase_  							=			tokenizer(UpperCamelCase__							,					padding=UpperCamelCase__							,					return_tensors=UpperCamelCase__       )
										self.assertIsInstance(UpperCamelCase__							,					UpperCamelCase__       )

										if FRAMEWORK != "jax":
															lowercase_  							=			list(batch.input_ids.numpy()[0]       )
										else:
															lowercase_  							=			list(batch.input_ids.tolist()[0]       )

										self.assertListEqual(UpperCamelCase__							,					UpperCamelCase__       )

										self.assertEqual((2, 37)							,					batch.input_ids.shape       )
										self.assertEqual((2, 37)							,					batch.attention_mask.shape       )

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										lowercase_  							=			self.ta_base_tokenizer
										lowercase_  							=			["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
										lowercase_  							=			tokenizer(UpperCamelCase__							,					padding=UpperCamelCase__							,					return_tensors=UpperCamelCase__       )
										# check if input_ids are returned and no decoder_input_ids
										self.assertIn("""input_ids"""							,					UpperCamelCase__       )
										self.assertIn("""attention_mask"""							,					UpperCamelCase__       )
										self.assertNotIn("""decoder_input_ids"""							,					UpperCamelCase__       )
										self.assertNotIn("""decoder_attention_mask"""							,					UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   Union[str, Any]       ):






										'''simple docstring'''






										lowercase_  							=			self.ta_base_tokenizer
										lowercase_  							=			[
										    """Summary of the text.""",
										    """Another summary.""",
										]
										lowercase_  							=			tokenizer(
										    text_target=UpperCamelCase__							,					max_length=32							,					padding="""max_length"""							,					truncation=UpperCamelCase__							,					return_tensors=UpperCamelCase__       )
										self.assertEqual(32							,					targets["""input_ids"""].shape[1]       )

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										lowercase_  							=			self.ta_base_tokenizer
										lowercase_  							=			["""A long paragraph for summarization. </s>"""]
										lowercase_  							=			["""Summary of the text. </s>"""]
										# fmt: off
										lowercase_  							=			[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
										lowercase_  							=			[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
										# fmt: on

										lowercase_  							=			tokenizer(UpperCamelCase__							,					text_target=UpperCamelCase__       )

										self.assertEqual(UpperCamelCase__							,					batch["""input_ids"""][0]       )
										self.assertEqual(UpperCamelCase__							,					batch["""labels"""][0]       )

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										lowercase_  							=			self.get_tokenizers()
										for tokenizer in tokenizers:
															with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																				self.assertNotEqual(tokenizer.model_max_length							,					42       )

        # Now let's start the test
										lowercase_  							=			self.get_tokenizers()
										for tokenizer in tokenizers:
															with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																				# Isolate this from the other tests because we save additional tokens/etc
																				lowercase_  							=			tempfile.mkdtemp()

																				lowercase_  							=			""" He is very happy, UNwant\u00E9d,running"""
																				lowercase_  							=			tokenizer.encode(UpperCamelCase__							,					add_special_tokens=UpperCamelCase__       )
																				tokenizer.save_pretrained(UpperCamelCase__       )

																				lowercase_  							=			tokenizer.__class__.from_pretrained(UpperCamelCase__       )
																				lowercase_  							=			after_tokenizer.encode(UpperCamelCase__							,					add_special_tokens=UpperCamelCase__       )
																				self.assertListEqual(UpperCamelCase__							,					UpperCamelCase__       )

																				shutil.rmtree(UpperCamelCase__       )

										lowercase_  							=			self.get_tokenizers(model_max_length=42       )
										for tokenizer in tokenizers:
															with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																				# Isolate this from the other tests because we save additional tokens/etc
																				lowercase_  							=			tempfile.mkdtemp()

																				lowercase_  							=			""" He is very happy, UNwant\u00E9d,running"""
																				tokenizer.add_tokens(["""bim""", """bambam"""]       )
																				lowercase_  							=			tokenizer.additional_special_tokens
																				additional_special_tokens.append("""new_additional_special_token"""       )
																				tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}       )
																				lowercase_  							=			tokenizer.encode(UpperCamelCase__							,					add_special_tokens=UpperCamelCase__       )
																				tokenizer.save_pretrained(UpperCamelCase__       )

																				lowercase_  							=			tokenizer.__class__.from_pretrained(UpperCamelCase__       )
																				lowercase_  							=			after_tokenizer.encode(UpperCamelCase__							,					add_special_tokens=UpperCamelCase__       )
																				self.assertListEqual(UpperCamelCase__							,					UpperCamelCase__       )
																				self.assertIn("""new_additional_special_token"""							,					after_tokenizer.additional_special_tokens       )
																				self.assertEqual(after_tokenizer.model_max_length							,					42       )

																				lowercase_  							=			tokenizer.__class__.from_pretrained(UpperCamelCase__							,					model_max_length=43       )
																				self.assertEqual(tokenizer.model_max_length							,					43       )

																				shutil.rmtree(UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   Tuple       ):






										'''simple docstring'''






										lowercase_  							=			[]
										if self.test_slow_tokenizer:
															tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

										if self.test_rust_tokenizer:
															tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

										for tokenizer_class, tokenizer_utils in tokenizer_list:
															with tempfile.TemporaryDirectory() as tmp_dir:
																				tokenizer_utils.save_pretrained(UpperCamelCase__       )

																				with open(os.path.join(UpperCamelCase__							,					"""special_tokens_map.json"""       )							,					encoding="""utf-8"""       ) as json_file:
																									lowercase_  							=			json.load(UpperCamelCase__       )

																				with open(os.path.join(UpperCamelCase__							,					"""tokenizer_config.json"""       )							,					encoding="""utf-8"""       ) as json_file:
																									lowercase_  							=			json.load(UpperCamelCase__       )

																				lowercase_  							=			[F'''<extra_id_{i}>''' for i in range(125       )]

																				lowercase_  							=			added_tokens_extra_ids + [
																				    """an_additional_special_token"""
																				]
																				lowercase_  							=			added_tokens_extra_ids + [
																				    """an_additional_special_token"""
																				]

																				with open(os.path.join(UpperCamelCase__							,					"""special_tokens_map.json"""       )							,					"""w"""							,					encoding="""utf-8"""       ) as outfile:
																									json.dump(UpperCamelCase__							,					UpperCamelCase__       )
																				with open(os.path.join(UpperCamelCase__							,					"""tokenizer_config.json"""       )							,					"""w"""							,					encoding="""utf-8"""       ) as outfile:
																									json.dump(UpperCamelCase__							,					UpperCamelCase__       )

																				# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																				# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																				# "special_tokens_map.json" files
																				lowercase_  							=			tokenizer_class.from_pretrained(
																				    UpperCamelCase__							,					)
																				self.assertIn(
																				    """an_additional_special_token"""							,					tokenizer_without_change_in_init.additional_special_tokens       )
																				# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																				self.assertEqual(
																				    ["""an_additional_special_token"""]							,					tokenizer_without_change_in_init.convert_ids_to_tokens(
																				        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]       )       )							,					)

																				# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																				lowercase_  							=			added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""							,					lstrip=UpperCamelCase__       )]
																				lowercase_  							=			tokenizer_class.from_pretrained(
																				    UpperCamelCase__							,					additional_special_tokens=UpperCamelCase__							,					)

																				self.assertIn("""a_new_additional_special_token"""							,					tokenizer.additional_special_tokens       )
																				self.assertEqual(
																				    ["""a_new_additional_special_token"""]							,					tokenizer.convert_ids_to_tokens(
																				        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]       )       )							,					)

					def   UpperCAmelCase__       (     self       :   str       ):






										'''simple docstring'''






										lowercase_  							=			[]
										if self.test_slow_tokenizer:
															tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

										if self.test_rust_tokenizer:
															tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

										for tokenizer_class, tokenizer_utils in tokenizer_list:
															with tempfile.TemporaryDirectory() as tmp_dir:
																				tokenizer_utils.save_pretrained(UpperCamelCase__       )

																				lowercase_  							=			tokenizer_class.from_pretrained(UpperCamelCase__       )

																				self.assertTrue(tokenizer.decode([255]       ) == """"""       )

					def   UpperCAmelCase__       (     self       :   int       ):






										'''simple docstring'''






										pass

					def   UpperCAmelCase__       (     self       :   Any       ):






										'''simple docstring'''






										pass

					def   UpperCAmelCase__       (     self       :   Optional[int]       ):






										'''simple docstring'''






										pass

					def   UpperCAmelCase__       (     self       :   int       ):






										'''simple docstring'''






										pass

					def   UpperCAmelCase__       (     self       :   Optional[int]       ):






										'''simple docstring'''






										lowercase_  							=			self.get_tokenizers(fast=UpperCamelCase__							,					do_lower_case=UpperCamelCase__       )
										for tokenizer in tokenizers:
															with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																				lowercase_  							=			["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
																				lowercase_  							=			tokenizer.convert_tokens_to_string(UpperCamelCase__       )

																				self.assertIsInstance(UpperCamelCase__							,					UpperCamelCase__       )







					def   UpperCAmelCase__       (     self       :   Union[str, Any]       ):






										'''simple docstring'''






										lowercase_  							=			self.get_tokenizers()
										for tokenizer in tokenizers:
															with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																				lowercase_  							=			[
																				    """bos_token""",
																				    """eos_token""",
																				    """unk_token""",
																				    """sep_token""",
																				    """pad_token""",
																				    """cls_token""",
																				    """mask_token""",
																				]

																				lowercase_  							=			0
																				lowercase_  							=			tokenizer.convert_ids_to_tokens(
																				    UpperCamelCase__							,					skip_special_tokens=UpperCamelCase__       )

																				for attr in attributes_list:
																									setattr(UpperCamelCase__							,					attr + """_id"""							,					UpperCamelCase__       )
																									self.assertEqual(getattr(UpperCamelCase__							,					UpperCamelCase__       )							,					UpperCamelCase__       )
																									self.assertEqual(getattr(UpperCamelCase__							,					attr + """_id"""       )							,					UpperCamelCase__       )

																									setattr(UpperCamelCase__							,					attr + """_id"""							,					UpperCamelCase__       )
																									self.assertEqual(getattr(UpperCamelCase__							,					UpperCamelCase__       )							,					UpperCamelCase__       )
																									self.assertEqual(getattr(UpperCamelCase__							,					attr + """_id"""       )							,					UpperCamelCase__       )

																				setattr(UpperCamelCase__							,					"""additional_special_tokens_ids"""							,					[]       )
																				self.assertListEqual(getattr(UpperCamelCase__							,					"""additional_special_tokens"""       )							,					[]       )
																				self.assertListEqual(getattr(UpperCamelCase__							,					"""additional_special_tokens_ids"""       )							,					[]       )

																				setattr(UpperCamelCase__							,					"""additional_special_tokens_ids"""							,					[token_id_to_test_setters]       )
																				self.assertListEqual(getattr(UpperCamelCase__							,					"""additional_special_tokens"""       )							,					[token_to_test_setters]       )
																				self.assertListEqual(getattr(UpperCamelCase__							,					"""additional_special_tokens_ids"""       )							,					[token_id_to_test_setters]       )






def 			UpperCAmelCase_				(      UpperCAmelCase__  ):
					if not isinstance(UpperCAmelCase__						,							UpperCAmelCase__  ):
										raise TypeError("""Input value must be an 'int' type"""  )

					lowercase_  							=			0
					while number:
										position += 1
										number >>= 1

					return position


if __name__ == "__main__":
							import doctest

							doctest.testmod()


