


import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCAmelCase			:  List[Any]     =get_tests_dir("""fixtures/test_sentencepiece.model""")







@require_sentencepiece
@require_tokenizers
class 				snake_case__(							UpperCAmelCase__,						unittest.TestCase							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  str					=							ReformerTokenizer
						SCREAMING_SNAKE_CASE__      :  List[str]					=							ReformerTokenizerFast
						SCREAMING_SNAKE_CASE__      :  Union[str, Any]					=							True
						SCREAMING_SNAKE_CASE__      :  List[Any]					=							False
						SCREAMING_SNAKE_CASE__      :  List[str]					=							True




						def 						lowercase_ (				self							)					->    Optional[Any]:
										super().setUp()

										lowerCAmelCase_ :      List[str]    				=				ReformerTokenizer(__lowercase   ,						keep_accents=__lowercase							)
										tokenizer.save_pretrained(self.tmpdirname							)




						def 						lowercase_ (				self							)					->    Tuple:
										lowerCAmelCase_ :      Any    				=				'''<s>'''
										lowerCAmelCase_ :      int    				=				1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowercase							)   ,						__lowercase							)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowercase							)   ,						__lowercase							)




						def 						lowercase_ (				self							)					->    int:
										lowerCAmelCase_ :      List[str]    				=				list(self.get_tokenizer().get_vocab().keys()							)

										self.assertEqual(vocab_keys[0]   ,						'''<unk>'''							)
										self.assertEqual(vocab_keys[1]   ,						'''<s>'''							)
										self.assertEqual(vocab_keys[-1]   ,						'''j'''							)
										self.assertEqual(len(__lowercase							)   ,						1_0_0_0							)




						def 						lowercase_ (				self							)					->    List[str]:
										self.assertEqual(self.get_tokenizer().vocab_size   ,						1_0_0_0							)




						def 						lowercase_ (				self							)					->    Any:
										if not self.test_rust_tokenizer:
														return

										lowerCAmelCase_ :      str    				=				self.get_tokenizer()
										lowerCAmelCase_ :      List[str]    				=				self.get_rust_tokenizer()

										lowerCAmelCase_ :      Dict    				=				'''I was born in 92000, and this is falsé.'''

										lowerCAmelCase_ :      Dict    				=				tokenizer.tokenize(__lowercase							)
										lowerCAmelCase_ :      Tuple    				=				rust_tokenizer.tokenize(__lowercase							)
										self.assertListEqual(__lowercase   ,						__lowercase							)

										lowerCAmelCase_ :      str    				=				tokenizer.encode(__lowercase   ,						add_special_tokens=__lowercase							)
										lowerCAmelCase_ :      Dict    				=				rust_tokenizer.encode(__lowercase   ,						add_special_tokens=__lowercase							)
										self.assertListEqual(__lowercase   ,						__lowercase							)

										lowerCAmelCase_ :      Optional[Any]    				=				self.get_rust_tokenizer()
										lowerCAmelCase_ :      Optional[int]    				=				tokenizer.encode(__lowercase							)
										lowerCAmelCase_ :      Optional[int]    				=				rust_tokenizer.encode(__lowercase							)
										self.assertListEqual(__lowercase   ,						__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase=1_5							)					->    Optional[int]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																		lowerCAmelCase_ :      List[Any]    				=				self.rust_tokenizer_class.from_pretrained(__lowercase   ,						**__lowercase							)

																		# Simple input
																		lowerCAmelCase_ :      Dict    				=				'''This is a simple input'''
																		lowerCAmelCase_ :      Any    				=				['''This is a simple input 1''', '''This is a simple input 2''']
																		lowerCAmelCase_ :      Any    				=				('''This is a simple input''', '''This is a pair''')
																		lowerCAmelCase_ :      Tuple    				=				[
																		    ('''This is a simple input 1''', '''This is a simple input 2'''),
																		    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																		]

																		# Simple input tests
																		self.assertRaises(__lowercase   ,						tokenizer_r.encode   ,						__lowercase   ,						max_length=__lowercase   ,						padding='''max_length'''							)

																		# Simple input
																		self.assertRaises(__lowercase   ,						tokenizer_r.encode_plus   ,						__lowercase   ,						max_length=__lowercase   ,						padding='''max_length'''							)

																		# Simple input
																		self.assertRaises(
																		    __lowercase   ,						tokenizer_r.batch_encode_plus   ,						__lowercase   ,						max_length=__lowercase   ,						padding='''max_length'''   ,						)

																		# Pair input
																		self.assertRaises(__lowercase   ,						tokenizer_r.encode   ,						__lowercase   ,						max_length=__lowercase   ,						padding='''max_length'''							)

																		# Pair input
																		self.assertRaises(__lowercase   ,						tokenizer_r.encode_plus   ,						__lowercase   ,						max_length=__lowercase   ,						padding='''max_length'''							)

																		# Pair input
																		self.assertRaises(
																		    __lowercase   ,						tokenizer_r.batch_encode_plus   ,						__lowercase   ,						max_length=__lowercase   ,						padding='''max_length'''   ,						)




						def 						lowercase_ (				self							)					->    str:
										pass




						def 						lowercase_ (				self							)					->    Optional[int]:
										lowerCAmelCase_ :      Any    				=				ReformerTokenizer(__lowercase   ,						keep_accents=__lowercase							)

										lowerCAmelCase_ :      List[Any]    				=				tokenizer.tokenize('''This is a test'''							)
										self.assertListEqual(__lowercase   ,						['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(__lowercase							)   ,						[2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]   ,						)

										lowerCAmelCase_ :      List[str]    				=				tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
										self.assertListEqual(
										    __lowercase   ,						[
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''s''',
										        '''é''',
										        '''.''',
										    ]   ,						)
										lowerCAmelCase_ :      Tuple    				=				tokenizer.convert_tokens_to_ids(__lowercase							)
										self.assertListEqual(
										    __lowercase   ,						[8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]   ,						)

										lowerCAmelCase_ :      Optional[int]    				=				tokenizer.convert_ids_to_tokens(__lowercase							)
										self.assertListEqual(
										    __lowercase   ,						[
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''<unk>''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''s''',
										        '''<unk>''',
										        '''.''',
										    ]   ,						)




						@cached_property
						def 						lowercase_ (				self							)					->    Optional[Any]:
										return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''							)




						@slow
						def 						lowercase_ (				self							)					->    int:
										lowerCAmelCase_ :      str    				=				'''Hello World!'''
										lowerCAmelCase_ :      str    				=				[1_2_6, 3_2, 2_6_2, 1_5_2, 3_8, 7_2, 2_8_7]

										self.assertListEqual(__lowercase   ,						self.big_tokenizer.encode(__lowercase							)							)




						@slow
						def 						lowercase_ (				self							)					->    Optional[int]:
										lowerCAmelCase_ :      Optional[int]    				=				(
										    '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
										    ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
										)
										lowerCAmelCase_ :      int    				=				[
										    1_0_8,
										    2_6_5,
										    2_4,
										    1_1_1,
										    4,
										    2_5_8,
										    1_5_6,
										    3_5,
										    2_8,
										    2_7_5,
										    3,
										    2_5_9,
										    2_9_7,
										    2_6_0,
										    8_4,
										    4,
										    3_5,
										    1_1_0,
										    4_4,
										    8,
										    2_5_9,
										    9_1,
										    2_6_8,
										    2_1,
										    1_1,
										    2_0_9,
										    2_7_4,
										    1_0_9,
										    2_6_6,
										    2_7_7,
										    1_1_7,
										    8_6,
										    9_3,
										    3_1_5,
										    2_5_8,
										    2_7_8,
										    2_5_8,
										    2_7_7,
										    2_5_8,
										    0,
										    2_5_8,
										    2_8_8,
										    2_5_8,
										    3_1_9,
										    2_5_8,
										    0,
										    2_5_8,
										    0,
										    2_5_8,
										    0,
										    2_5_8,
										    0,
										    2_5_8,
										    2_8_7,
										    2_5_8,
										    3_1_5,
										    2_5_8,
										    2_8_9,
										    2_5_8,
										    2_7_8,
										    9_9,
										    2_6_9,
										    2_6_6,
										    2_6_2,
										    8,
										    2_5_9,
										    2_4_1,
										    4,
										    2_1_7,
										    2_3_0,
										    2_6_8,
										    2_6_6,
										    5_5,
										    1_6_8,
										    1_0_6,
										    7_5,
										    1_9_3,
										    2_6_6,
										    2_2_3,
										    2_7,
										    4_9,
										    2_6,
										    2_8_2,
										    2_5,
										    2_6_4,
										    2_9_9,
										    1_9,
										    2_6,
										    0,
										    2_5_8,
										    2_7_7,
										    1_1_7,
										    8_6,
										    9_3,
										    1_7_6,
										    1_8_3,
										    2_7_0,
										    1_1,
										    2_6_2,
										    4_2,
										    6_1,
										    2_6_5,
										]

										self.assertListEqual(__lowercase   ,						self.big_tokenizer.encode(__lowercase							)							)




						@require_torch
						@slow
						def 						lowercase_ (				self							)					->    Any:
										import torch

										from transformers import ReformerConfig, ReformerModel

										# Build sequence
										lowerCAmelCase_ :      Optional[Any]    				=				list(self.big_tokenizer.get_vocab().keys()							)[:1_0]
										lowerCAmelCase_ :      Optional[int]    				=				''' '''.join(__lowercase							)
										lowerCAmelCase_ :      Union[str, Any]    				=				self.big_tokenizer.encode_plus(__lowercase   ,						return_tensors='''pt'''							)
										lowerCAmelCase_ :      List[Any]    				=				self.big_tokenizer.batch_encode_plus([sequence, sequence]   ,						return_tensors='''pt'''							)

										lowerCAmelCase_ :      Union[str, Any]    				=				ReformerConfig()
										# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
										lowerCAmelCase_ :      List[str]    				=				encoded_sequence['''input_ids'''].shape
										lowerCAmelCase_ :      Union[str, Any]    				=				ReformerModel(__lowercase							)

										# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
										assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

										with torch.no_grad():
														model(**__lowercase							)
														model(**__lowercase							)




						@slow
						def 						lowercase_ (				self							)					->    List[Any]:
										# fmt: off
										lowerCAmelCase_ :      int    				=				{'''input_ids''': [[1_0_8, 2_6_5, 2_4, 1_1_1, 4, 2_5_8, 1_5_6, 7, 5_1, 2_7_9, 5_8, 7, 7_6, 2_5, 6_9, 2_7_8], [1_4_0, 2_4_3, 2_6_4, 1_3_4, 1_7, 2_6_7, 7_7, 2_6_3, 2_2, 2_6_2, 2_9_7, 2_5_8, 3_0_4, 1_7_7, 2_7_9, 2_6_6, 1_4, 8_9, 1_3, 3_5, 2_6_1, 2_9_9, 2_7_2, 1_3_7, 2_7_5, 2_7_8]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
										# fmt: on

										# This tokenizer does not know some characters like ")".
										# That is the reason why we use very simple texts here.
										# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
										lowerCAmelCase_ :      int    				=				[
										    '''This is a very simple sentence.''',
										    '''The quick brown fox jumps over the lazy dog.''',
										]

										self.tokenizer_integration_test_util(
										    expected_encoding=__lowercase   ,						model_name='''google/reformer-crime-and-punishment'''   ,						revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a'''   ,						padding=__lowercase   ,						sequences=__lowercase   ,						)



from manim import *







class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						def 						lowercase_ (				self							)					->    Tuple:
										lowerCAmelCase_ :      Dict    				=				Rectangle(height=0.5   ,						width=0.5							)
										lowerCAmelCase_ :      Tuple    				=				Rectangle(height=0.25   ,						width=0.25							)
										lowerCAmelCase_ :      Tuple    				=				Rectangle(height=0.46   ,						width=0.46							).set_stroke(width=0							)

										lowerCAmelCase_ :      Optional[int]    				=				[mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      int    				=				[mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      Optional[int]    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      List[str]    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      int    				=				VGroup(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      Tuple    				=				Text('''CPU'''   ,						font_size=2_4							)
										lowerCAmelCase_ :      Union[str, Any]    				=				Group(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0.5   ,						aligned_edge=__lowercase							)
										cpu.move_to([-2.5, -0.5, 0]							)
										self.add(__lowercase							)

										lowerCAmelCase_ :      List[str]    				=				[mem.copy() for i in range(4							)]
										lowerCAmelCase_ :      Any    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      List[Any]    				=				Text('''GPU'''   ,						font_size=2_4							)
										lowerCAmelCase_ :      int    				=				Group(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0.5   ,						aligned_edge=__lowercase							)
										gpu.move_to([-1, -1, 0]							)
										self.add(__lowercase							)

										lowerCAmelCase_ :      str    				=				[mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      Dict    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)

										lowerCAmelCase_ :      Dict    				=				Text('''Model'''   ,						font_size=2_4							)
										lowerCAmelCase_ :      str    				=				Group(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0.5   ,						aligned_edge=__lowercase							)
										model.move_to([3, -1.0, 0]							)
										self.add(__lowercase							)

										lowerCAmelCase_ :      int    				=				[]
										lowerCAmelCase_ :      int    				=				[]
										lowerCAmelCase_ :      Dict    				=				[]

										for i, rect in enumerate(__lowercase							):
														rect.set_stroke(__lowercase							)

														lowerCAmelCase_ :      Any    				=				Rectangle(height=0.46 / 4   ,						width=0.46 / 3							).set_stroke(width=0.0							).set_fill(__lowercase   ,						opacity=0.7							)

														if i == 0:
																		cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT							)   ,						buff=0.02   ,						direction=__lowercase							)
																		cpu_target.set_x(cpu_target.get_x() + 0.1							)
														elif i == 3:
																		cpu_target.next_to(model_cpu_arr[0]   ,						direction=__lowercase   ,						buff=0.0							)
														else:
																		cpu_target.next_to(model_cpu_arr[i - 1]   ,						direction=__lowercase   ,						buff=0.0							)
														self.add(__lowercase							)
														model_cpu_arr.append(__lowercase							)

										self.add(*__lowercase   ,						*__lowercase   ,						*__lowercase							)

										lowerCAmelCase_ :      Union[str, Any]    				=				[mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      List[str]    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)

										lowerCAmelCase_ :      Union[str, Any]    				=				Text('''Loaded Checkpoint'''   ,						font_size=2_4							)
										lowerCAmelCase_ :      int    				=				Group(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0.5   ,						aligned_edge=__lowercase							)
										checkpoint.move_to([3, 0.5, 0]							)

										self.add(__lowercase							)

										lowerCAmelCase_ :      Optional[Any]    				=				[]
										lowerCAmelCase_ :      Dict    				=				[]

										for i, rect in enumerate(__lowercase							):
														lowerCAmelCase_ :      str    				=				fill.copy().set_fill(__lowercase   ,						opacity=0.7							)
														target.move_to(__lowercase							)
														ckpt_arr.append(__lowercase							)

														lowerCAmelCase_ :      Union[str, Any]    				=				target.copy()
														if i < 5:
																		cpu_target.move_to(cpu_left_col_base[i + 1]							)
														else:
																		cpu_target.move_to(cpu_right_col_base[i - 5]							)
														ckpt_cpu_arr.append(__lowercase							)
										self.add(*__lowercase   ,						*__lowercase							)

										lowerCAmelCase_ :      Union[str, Any]    				=				Square(side_length=2.2							)
										key.move_to([-5, 2, 0]							)

										lowerCAmelCase_ :      str    				=				MarkupText(
										    f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""   ,						font_size=1_8   ,						)

										key_text.move_to([-5, 2.4, 0]							)

										self.add(__lowercase   ,						__lowercase							)

										lowerCAmelCase_ :      str    				=				MarkupText(
										    f"""<span fgcolor='{BLUE}'>●</span> Checkpoint"""   ,						font_size=1_8   ,						)

										blue_text.next_to(__lowercase   ,						DOWN * 2.4   ,						aligned_edge=key_text.get_left()							)
										self.add(__lowercase							)

										lowerCAmelCase_ :      str    				=				MarkupText(
										    f"""Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device."""   ,						font_size=2_4   ,						)
										step_a.move_to([2, 2, 0]							)

										lowerCAmelCase_ :      List[Any]    				=				[meta_mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      Any    				=				[meta_mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      Any    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      Union[str, Any]    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      int    				=				VGroup(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      List[str]    				=				Text('''Disk'''   ,						font_size=2_4							)
										lowerCAmelCase_ :      Optional[int]    				=				Group(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0.5   ,						aligned_edge=__lowercase							)
										disk.move_to([-4.0, -1.25, 0]							)
										self.play(Write(__lowercase   ,						run_time=3							)   ,						Write(__lowercase   ,						run_time=1							)   ,						Create(__lowercase   ,						run_time=1							)							)

										lowerCAmelCase_ :      int    				=				[]
										for i, rect in enumerate(__lowercase							):
														lowerCAmelCase_ :      int    				=				rect.copy()
														target.generate_target()
														target.target.move_to(disk_left_col_base[i]							).scale(0.5							)
														animations.append(MoveToTarget(__lowercase   ,						run_time=1.5							)							)
										self.play(*__lowercase							)

										self.play(FadeOut(__lowercase							)							)

										lowerCAmelCase_ :      Union[str, Any]    				=				MarkupText(f"""Then, the checkpoint is removed from memory\nthrough garbage collection."""   ,						font_size=2_4							)
										step_a.move_to([2, 2, 0]							)

										self.play(Write(__lowercase   ,						run_time=3							)							)

										self.play(
										    FadeOut(__lowercase   ,						__lowercase   ,						*__lowercase   ,						*__lowercase							)   ,						)

										self.wait()