
import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
							lowerCamelCase_						: List[str]         =  """pt"""
elif is_tf_available():
							lowerCamelCase_						: Any         =  """tf"""
else:
							lowerCamelCase_						: Union[str, Any]         =  """jax"""

class        _UpperCamelCase				(					_A     ,       unittest.TestCase  ):






			'''simple docstring'''

			__UpperCamelCase      :						Tuple					      = PerceiverTokenizer
			__UpperCamelCase      :						Optional[Any]					      = False

			def       lowerCAmelCase__	(       self :       Any      ):
							super().setUp()
							UpperCamelCase_:							Dict	=						PerceiverTokenizer()
							tokenizer.save_pretrained(self.tmpdirname      )

			@cached_property
			def       lowerCAmelCase__	(       self :       int      ):
							return PerceiverTokenizer.from_pretrained("""deepmind/language-perceiver"""      )

			def       lowerCAmelCase__	(       self :       Optional[int]						,	**snake_case_ :       Optional[int]      ):
							return self.tokenizer_class.from_pretrained(self.tmpdirname						,	**__UpperCamelCase      )

			def       lowerCAmelCase__	(       self :       int						,	snake_case_ :       int						,	snake_case_ :       Union[str, Any]=False						,	snake_case_ :       Optional[int]=20						,	snake_case_ :       int=5      ):
							# XXX The default common tokenizer tests assume that every ID is decodable on its own.
							# This assumption is invalid for Perceiver because single bytes might not be
							# valid utf-8 (byte 128 for instance).
							# Here we're overriding the smallest possible method to provide
							# a clean sequence without making the same assumption.

							UpperCamelCase_:							Dict	=						[]
							for i in range(len(__UpperCamelCase      )      ):
											try:
															UpperCamelCase_:							List[Any]	=						tokenizer.decode([i]						,	clean_up_tokenization_spaces=__UpperCamelCase      )
											except UnicodeDecodeError:
															pass
											toks.append((i, tok)      )

							UpperCamelCase_:							Optional[int]	=						list(filter(lambda snake_case_      : re.match(R"""^[ a-zA-Z]+$"""						,	t[1]      )						,	__UpperCamelCase      )      )
							UpperCamelCase_:							str	=						list(filter(lambda snake_case_      : [t[0]] == tokenizer.encode(t[1]						,	add_special_tokens=__UpperCamelCase      )						,	__UpperCamelCase      )      )
							if max_length is not None and len(__UpperCamelCase      ) > max_length:
											UpperCamelCase_:							int	=						toks[:max_length]
							if min_length is not None and len(__UpperCamelCase      ) < min_length and len(__UpperCamelCase      ) > 0:
											while len(__UpperCamelCase      ) < min_length:
															UpperCamelCase_:							Dict	=						toks + toks
        # toks_str = [t[1] for t in toks]
							UpperCamelCase_:							Any	=						[t[0] for t in toks]

							# Ensure consistency
							UpperCamelCase_:							str	=						tokenizer.decode(__UpperCamelCase						,	clean_up_tokenization_spaces=__UpperCamelCase      )
							if " " not in output_txt and len(__UpperCamelCase      ) > 1:
											UpperCamelCase_:							Optional[Any]	=						(
											    tokenizer.decode([toks_ids[0]]						,	clean_up_tokenization_spaces=__UpperCamelCase      )
											    + """ """
											    + tokenizer.decode(toks_ids[1:]						,	clean_up_tokenization_spaces=__UpperCamelCase      )
											)
							if with_prefix_space:
											UpperCamelCase_:							List[str]	=						""" """ + output_txt
							UpperCamelCase_:							Dict	=						tokenizer.encode(__UpperCamelCase						,	add_special_tokens=__UpperCamelCase      )
							return output_txt, output_ids

			def       lowerCAmelCase__	(       self :       Optional[int]      ):
							UpperCamelCase_:							List[Any]	=						self.perceiver_tokenizer
							UpperCamelCase_:							Optional[Any]	=						"""Unicode €."""
							UpperCamelCase_:							Dict	=						tokenizer(__UpperCamelCase      )
							UpperCamelCase_:							int	=						[4, 91, 116, 111, 105, 117, 106, 107, 38, 232, 136, 178, 52, 5]
							self.assertEqual(encoded["""input_ids"""]						,	__UpperCamelCase      )

							# decoding
							UpperCamelCase_:							int	=						tokenizer.decode(__UpperCamelCase      )
							self.assertEqual(__UpperCamelCase						,	"""[CLS]Unicode €.[SEP]"""      )

							UpperCamelCase_:							Any	=						tokenizer("""e è é ê ë"""      )
							UpperCamelCase_:							Optional[Any]	=						[4, 107, 38, 201, 174, 38, 201, 175, 38, 201, 176, 38, 201, 177, 5]
							self.assertEqual(encoded["""input_ids"""]						,	__UpperCamelCase      )
							# decoding
							UpperCamelCase_:							Union[str, Any]	=						tokenizer.decode(__UpperCamelCase      )
							self.assertEqual(__UpperCamelCase						,	"""[CLS]e è é ê ë[SEP]"""      )

							# encode/decode, but with `encode` instead of `__call__`
							self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""      )      )						,	"""[CLS]e è é ê ë[SEP]"""      )

			def       lowerCAmelCase__	(       self :       str      ):
							UpperCamelCase_:							int	=						self.perceiver_tokenizer
							UpperCamelCase_:							Union[str, Any]	=						["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
							# fmt: off
							UpperCamelCase_:							Optional[Any]	=						[4, 71, 38, 114, 117, 116, 109, 38, 118, 103, 120, 103, 109, 120, 103, 118, 110, 38, 108, 117, 120, 38, 121, 123, 115, 115, 103, 120, 111, 128, 103, 122, 111, 117, 116, 52, 5, 0]
							# fmt: on
							UpperCamelCase_:							Union[str, Any]	=						tokenizer(__UpperCamelCase						,	padding=__UpperCamelCase						,	return_tensors=__UpperCamelCase      )
							self.assertIsInstance(__UpperCamelCase						,	__UpperCamelCase      )

							if FRAMEWORK != "jax":
											UpperCamelCase_:							Tuple	=						list(batch.input_ids.numpy()[0]      )
							else:
											UpperCamelCase_:							Optional[int]	=						list(batch.input_ids.tolist()[0]      )

							self.assertListEqual(__UpperCamelCase						,	__UpperCamelCase      )

							self.assertEqual((2, 38)						,	batch.input_ids.shape      )
							self.assertEqual((2, 38)						,	batch.attention_mask.shape      )

			def       lowerCAmelCase__	(       self :       List[Any]      ):
							UpperCamelCase_:							Optional[Any]	=						self.perceiver_tokenizer
							UpperCamelCase_:							Tuple	=						["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
							UpperCamelCase_:							str	=						tokenizer(__UpperCamelCase						,	padding=__UpperCamelCase						,	return_tensors=__UpperCamelCase      )
							# check if input_ids are returned and no decoder_input_ids
							self.assertIn("""input_ids"""						,	__UpperCamelCase      )
							self.assertIn("""attention_mask"""						,	__UpperCamelCase      )
							self.assertNotIn("""decoder_input_ids"""						,	__UpperCamelCase      )
							self.assertNotIn("""decoder_attention_mask"""						,	__UpperCamelCase      )

			def       lowerCAmelCase__	(       self :       int      ):
							UpperCamelCase_:							Optional[int]	=						self.perceiver_tokenizer
							UpperCamelCase_:							Optional[int]	=						[
							    """Summary of the text.""",
							    """Another summary.""",
							]
							UpperCamelCase_:							Tuple	=						tokenizer(
							    text_target=__UpperCamelCase						,	max_length=32						,	padding="""max_length"""						,	truncation=__UpperCamelCase						,	return_tensors=__UpperCamelCase      )
							self.assertEqual(32						,	targets["""input_ids"""].shape[1]      )

			def       lowerCAmelCase__	(       self :       Tuple      ):
							# safety check on max_len default value so we are sure the test works
							UpperCamelCase_:							Union[str, Any]	=						self.get_tokenizers()
							for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}'''      ):
															self.assertNotEqual(tokenizer.model_max_length						,	42      )

        # Now let's start the test
							UpperCamelCase_:							List[Any]	=						self.get_tokenizers()
							for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}'''      ):
															# Isolate this from the other tests because we save additional tokens/etc
															UpperCamelCase_:							int	=						tempfile.mkdtemp()

															UpperCamelCase_:							str	=						""" He is very happy, UNwant\u00E9d,running"""
															UpperCamelCase_:							Optional[int]	=						tokenizer.encode(__UpperCamelCase						,	add_special_tokens=__UpperCamelCase      )
															tokenizer.save_pretrained(__UpperCamelCase      )

															UpperCamelCase_:							Optional[int]	=						tokenizer.__class__.from_pretrained(__UpperCamelCase      )
															UpperCamelCase_:							List[Any]	=						after_tokenizer.encode(__UpperCamelCase						,	add_special_tokens=__UpperCamelCase      )
															self.assertListEqual(__UpperCamelCase						,	__UpperCamelCase      )

															shutil.rmtree(__UpperCamelCase      )

							UpperCamelCase_:							Optional[Any]	=						self.get_tokenizers(model_max_length=42      )
							for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}'''      ):
															# Isolate this from the other tests because we save additional tokens/etc
															UpperCamelCase_:							List[Any]	=						tempfile.mkdtemp()

															UpperCamelCase_:							Any	=						""" He is very happy, UNwant\u00E9d,running"""
															tokenizer.add_tokens(["""bim""", """bambam"""]      )
															UpperCamelCase_:							Dict	=						tokenizer.additional_special_tokens
															additional_special_tokens.append("""new_additional_special_token"""      )
															tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}      )
															UpperCamelCase_:							List[Any]	=						tokenizer.encode(__UpperCamelCase						,	add_special_tokens=__UpperCamelCase      )
															tokenizer.save_pretrained(__UpperCamelCase      )

															UpperCamelCase_:							str	=						tokenizer.__class__.from_pretrained(__UpperCamelCase      )
															UpperCamelCase_:							List[str]	=						after_tokenizer.encode(__UpperCamelCase						,	add_special_tokens=__UpperCamelCase      )
															self.assertListEqual(__UpperCamelCase						,	__UpperCamelCase      )
															self.assertIn("""new_additional_special_token"""						,	after_tokenizer.additional_special_tokens      )
															self.assertEqual(after_tokenizer.model_max_length						,	42      )

															UpperCamelCase_:							List[str]	=						tokenizer.__class__.from_pretrained(__UpperCamelCase						,	model_max_length=43      )
															self.assertEqual(tokenizer.model_max_length						,	43      )

															shutil.rmtree(__UpperCamelCase      )

			def       lowerCAmelCase__	(       self :       Optional[int]      ):
							UpperCamelCase_:							int	=						[]
							if self.test_slow_tokenizer:
											tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

							if self.test_rust_tokenizer:
											tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

							for tokenizer_class, tokenizer_utils in tokenizer_list:
											with tempfile.TemporaryDirectory() as tmp_dir:
															tokenizer_utils.save_pretrained(__UpperCamelCase      )

															with open(os.path.join(__UpperCamelCase						,	"""special_tokens_map.json"""      )						,	encoding="""utf-8"""      ) as json_file:
																			UpperCamelCase_:							List[str]	=						json.load(__UpperCamelCase      )

															with open(os.path.join(__UpperCamelCase						,	"""tokenizer_config.json"""      )						,	encoding="""utf-8"""      ) as json_file:
																			UpperCamelCase_:							Dict	=						json.load(__UpperCamelCase      )

															UpperCamelCase_:							int	=						[f'''<extra_id_{i}>''' for i in range(125      )]

															UpperCamelCase_:							int	=						added_tokens_extra_ids + [
															    """an_additional_special_token"""
															]
															UpperCamelCase_:							int	=						added_tokens_extra_ids + [
															    """an_additional_special_token"""
															]

															with open(os.path.join(__UpperCamelCase						,	"""special_tokens_map.json"""      )						,	"""w"""						,	encoding="""utf-8"""      ) as outfile:
																			json.dump(__UpperCamelCase						,	__UpperCamelCase      )
															with open(os.path.join(__UpperCamelCase						,	"""tokenizer_config.json"""      )						,	"""w"""						,	encoding="""utf-8"""      ) as outfile:
																			json.dump(__UpperCamelCase						,	__UpperCamelCase      )

															# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
															# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
															# "special_tokens_map.json" files
															UpperCamelCase_:							Any	=						tokenizer_class.from_pretrained(
															    __UpperCamelCase						,	)
															self.assertIn(
															    """an_additional_special_token"""						,	tokenizer_without_change_in_init.additional_special_tokens      )
															self.assertEqual(
															    ["""an_additional_special_token"""]						,	tokenizer_without_change_in_init.convert_ids_to_tokens(
															        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]      )      )						,	)

															# Now we test that we can change the value of additional_special_tokens in the from_pretrained
															UpperCamelCase_:							Any	=						added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""						,	lstrip=__UpperCamelCase      )]
															UpperCamelCase_:							str	=						tokenizer_class.from_pretrained(
															    __UpperCamelCase						,	additional_special_tokens=__UpperCamelCase						,	)

															self.assertIn("""a_new_additional_special_token"""						,	tokenizer.additional_special_tokens      )
															self.assertEqual(
															    ["""a_new_additional_special_token"""]						,	tokenizer.convert_ids_to_tokens(
															        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]      )      )						,	)

			def       lowerCAmelCase__	(       self :       int      ):
							UpperCamelCase_:							List[str]	=						self.perceiver_tokenizer
							self.assertEqual(tokenizer.decode([178]      )						,	"""�"""      )

			def       lowerCAmelCase__	(       self :       List[str]      ):
							pass

			def       lowerCAmelCase__	(       self :       Tuple      ):
							pass

			def       lowerCAmelCase__	(       self :       Tuple      ):
							pass

			def       lowerCAmelCase__	(       self :       Optional[Any]      ):
							pass




			def       lowerCAmelCase__	(       self :       int      ):
							# The default common tokenizer tests uses invalid tokens for Perceiver that can only accept one-character
							# strings and special added tokens as tokens
							UpperCamelCase_:							Optional[Any]	=						self.get_tokenizers(fast=__UpperCamelCase						,	do_lower_case=__UpperCamelCase      )
							for tokenizer in tokenizers:
											with self.subTest(f'''{tokenizer.__class__.__name__}'''      ):
															UpperCamelCase_:							List[str]	=						["""[CLS]""", """t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """s""", """t""", """[SEP]"""]
															UpperCamelCase_:							Optional[Any]	=						tokenizer.convert_tokens_to_string(__UpperCamelCase      )

															self.assertIsInstance(__UpperCamelCase						,	__UpperCamelCase      )








def        A__					(						lowerCamelCase = 50						)  ->	int:
				UpperCamelCase_:							List[Any]	=						[[0] * 3 for _ in range(length + 1						)]

				for row_length in range(length + 1						):
								for tile_length in range(2     ,    5						):
												for tile_start in range(row_length - tile_length + 1						):
																different_colour_ways_number[row_length][tile_length - 2] += (
																    different_colour_ways_number[row_length - tile_start - tile_length][
																        tile_length - 2
																    ]
																    + 1
																)

				return sum(different_colour_ways_number[length]						)


if __name__ == "__main__":
							print(F"""{solution() = }""")






