from manim import *
class        snake_case (				__UpperCAmelCase       ):




					"""simple docstring"""

					def 	__lowerCAmelCase		(       self     :			Any    ):
									UpperCAmelCase__    =							Rectangle(height=0.5					,width=0.5    )
									UpperCAmelCase__    =							Rectangle(height=0.2_5					,width=0.2_5    )
									UpperCAmelCase__    =							Rectangle(height=0.4_6					,width=0.4_6    ).set_stroke(width=0    )

									UpperCAmelCase__    =							[mem.copy() for i in range(6    )]
									UpperCAmelCase__    =							[mem.copy() for i in range(6    )]
									UpperCAmelCase__    =							VGroup(*lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0    )
									UpperCAmelCase__    =							VGroup(*lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0    )
									UpperCAmelCase__    =							VGroup(lowerCamelCase__					,lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0    )
									UpperCAmelCase__    =							Text('CPU'					,font_size=24    )
									UpperCAmelCase__    =							Group(lowerCamelCase__					,lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0.5					,aligned_edge=lowerCamelCase__    )
									cpu.move_to([-2.5, -0.5, 0]    )
									self.add(lowerCamelCase__    )

									UpperCAmelCase__    =							[mem.copy() for i in range(4    )]
									UpperCAmelCase__    =							VGroup(*lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0    )
									UpperCAmelCase__    =							Text('GPU'					,font_size=24    )
									UpperCAmelCase__    =							Group(lowerCamelCase__					,lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0.5					,aligned_edge=lowerCamelCase__    )
									gpu.move_to([-1, -1, 0]    )
									self.add(lowerCamelCase__    )

									UpperCAmelCase__    =							[mem.copy() for i in range(6    )]
									UpperCAmelCase__    =							VGroup(*lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0    )

									UpperCAmelCase__    =							Text('Model'					,font_size=24    )
									UpperCAmelCase__    =							Group(lowerCamelCase__					,lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0.5					,aligned_edge=lowerCamelCase__    )
									model.move_to([3, -1.0, 0]    )
									self.add(lowerCamelCase__    )

									UpperCAmelCase__    =							[]
									UpperCAmelCase__    =							[]
									UpperCAmelCase__    =							[]

									for i, rect in enumerate(lowerCamelCase__    ):
													rect.set_stroke(lowerCamelCase__    )

													UpperCAmelCase__    =							Rectangle(height=0.4_6 / 4					,width=0.4_6 / 3    ).set_stroke(width=0.0    ).set_fill(lowerCamelCase__					,opacity=0.7    )

													if i == 0:
																	cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT    )					,buff=0.0_2					,direction=lowerCamelCase__    )
																	cpu_target.set_x(cpu_target.get_x() + 0.1    )
													elif i == 3:
																	cpu_target.next_to(model_cpu_arr[0]					,direction=lowerCamelCase__					,buff=0.0    )
													else:
																	cpu_target.next_to(model_cpu_arr[i - 1]					,direction=lowerCamelCase__					,buff=0.0    )
													self.add(lowerCamelCase__    )
													model_cpu_arr.append(lowerCamelCase__    )

									self.add(*lowerCamelCase__					,*lowerCamelCase__					,*lowerCamelCase__    )

									UpperCAmelCase__    =							[mem.copy() for i in range(6    )]
									UpperCAmelCase__    =							VGroup(*lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0    )

									UpperCAmelCase__    =							Text('Loaded Checkpoint'					,font_size=24    )
									UpperCAmelCase__    =							Group(lowerCamelCase__					,lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0.5					,aligned_edge=lowerCamelCase__    )
									checkpoint.move_to([3, 0.5, 0]    )

									self.add(lowerCamelCase__    )

									UpperCAmelCase__    =							[]
									UpperCAmelCase__    =							[]

									for i, rect in enumerate(lowerCamelCase__    ):
													UpperCAmelCase__    =							fill.copy().set_fill(lowerCamelCase__					,opacity=0.7    )
													target.move_to(lowerCamelCase__    )
													ckpt_arr.append(lowerCamelCase__    )

													UpperCAmelCase__    =							target.copy()
													if i < 5:
																	cpu_target.move_to(cpu_left_col_base[i + 1]    )
													else:
																	cpu_target.move_to(cpu_right_col_base[i - 5]    )
													ckpt_cpu_arr.append(lowerCamelCase__    )
									self.add(*lowerCamelCase__					,*lowerCamelCase__    )

									UpperCAmelCase__    =							Square(side_length=2.2    )
									key.move_to([-5, 2, 0]    )

									UpperCAmelCase__    =							MarkupText(
									    f'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''					,font_size=18					,)

									key_text.move_to([-5, 2.4, 0]    )

									self.add(lowerCamelCase__					,lowerCamelCase__    )

									UpperCAmelCase__    =							MarkupText(
									    f'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''					,font_size=18					,)

									blue_text.next_to(lowerCamelCase__					,DOWN * 2.4					,aligned_edge=key_text.get_left()    )
									self.add(lowerCamelCase__    )

									UpperCAmelCase__    =							MarkupText(
									    f'''Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device.'''					,font_size=24					,)
									step_a.move_to([2, 2, 0]    )

									UpperCAmelCase__    =							[meta_mem.copy() for i in range(6    )]
									UpperCAmelCase__    =							[meta_mem.copy() for i in range(6    )]
									UpperCAmelCase__    =							VGroup(*lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0    )
									UpperCAmelCase__    =							VGroup(*lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0    )
									UpperCAmelCase__    =							VGroup(lowerCamelCase__					,lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0    )
									UpperCAmelCase__    =							Text('Disk'					,font_size=24    )
									UpperCAmelCase__    =							Group(lowerCamelCase__					,lowerCamelCase__    ).arrange(lowerCamelCase__					,buff=0.5					,aligned_edge=lowerCamelCase__    )
									disk.move_to([-4.0, -1.2_5, 0]    )
									self.play(Write(lowerCamelCase__					,run_time=3    )					,Write(lowerCamelCase__					,run_time=1    )					,Create(lowerCamelCase__					,run_time=1    )    )

									UpperCAmelCase__    =							[]
									for i, rect in enumerate(lowerCamelCase__    ):
													UpperCAmelCase__    =							rect.copy()
													target.generate_target()
													target.target.move_to(disk_left_col_base[i]    ).scale(0.5    )
													animations.append(MoveToTarget(lowerCamelCase__					,run_time=1.5    )    )
									self.play(*lowerCamelCase__    )

									self.play(FadeOut(lowerCamelCase__    )    )

									UpperCAmelCase__    =							MarkupText(f'''Then, the checkpoint is removed from memory\nthrough garbage collection.'''					,font_size=24    )
									step_a.move_to([2, 2, 0]    )

									self.play(Write(lowerCamelCase__					,run_time=3    )    )

									self.play(
									    FadeOut(lowerCamelCase__					,lowerCamelCase__					,*lowerCamelCase__					,*lowerCamelCase__    )					,)

									self.wait()




"""simple docstring"""






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase__						:						Any		  =			get_tests_dir('fixtures/test_sentencepiece.model')
@require_sentencepiece
class        snake_case (				__UpperCAmelCase   ,					unittest.TestCase       ):




					"""simple docstring"""

					snake_case__       					=				XLMProphetNetTokenizer
					snake_case__       					=				False
					snake_case__       					=				True







					def 	__lowerCAmelCase		(       self     :			Any    ):
									super().setUp()

									# We have a SentencePiece fixture for testing
									UpperCAmelCase__    =							XLMProphetNetTokenizer(lowerCamelCase__					,keep_accents=lowerCamelCase__    )
									tokenizer.save_pretrained(self.tmpdirname    )







					def 	__lowerCAmelCase		(       self     :			Dict    ):
									UpperCAmelCase__    =							'[PAD]'
									UpperCAmelCase__    =							0

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase__    )					,lowerCamelCase__    )
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase__    )					,lowerCamelCase__    )







					def 	__lowerCAmelCase		(       self     :			List[str]    ):
									UpperCAmelCase__    =							list(self.get_tokenizer().get_vocab().keys()    )

									self.assertEqual(vocab_keys[0]					,'[PAD]'    )
									self.assertEqual(vocab_keys[1]					,'[CLS]'    )
									self.assertEqual(vocab_keys[-1]					,'j'    )
									self.assertEqual(len(lowerCamelCase__    )					,1_012    )







					def 	__lowerCAmelCase		(       self     :			List[Any]    ):
									self.assertEqual(self.get_tokenizer().vocab_size					,1_012    )







					def 	__lowerCAmelCase		(       self     :			str    ):
									UpperCAmelCase__    =							XLMProphetNetTokenizer(lowerCamelCase__					,keep_accents=lowerCamelCase__    )

									UpperCAmelCase__    =							tokenizer.tokenize('This is a test'    )
									self.assertListEqual(lowerCamelCase__					,['▁This', '▁is', '▁a', '▁t', 'est']    )

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(lowerCamelCase__    )					,[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					,)

									UpperCAmelCase__    =							tokenizer.tokenize('I was born in 92000, and this is falsé.'    )
									self.assertListEqual(
									    lowerCamelCase__					,[
									        SPIECE_UNDERLINE + 'I',
									        SPIECE_UNDERLINE + 'was',
									        SPIECE_UNDERLINE + 'b',
									        'or',
									        'n',
									        SPIECE_UNDERLINE + 'in',
									        SPIECE_UNDERLINE + '',
									        '9',
									        '2',
									        '0',
									        '0',
									        '0',
									        ',',
									        SPIECE_UNDERLINE + 'and',
									        SPIECE_UNDERLINE + 'this',
									        SPIECE_UNDERLINE + 'is',
									        SPIECE_UNDERLINE + 'f',
									        'al',
									        's',
									        'é',
									        '.',
									    ]					,)
									UpperCAmelCase__    =							tokenizer.convert_tokens_to_ids(lowerCamelCase__    )
									self.assertListEqual(
									    lowerCamelCase__					,[
									        value + tokenizer.fairseq_offset
									        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
									    ]					,)

									UpperCAmelCase__    =							tokenizer.convert_ids_to_tokens(lowerCamelCase__    )
									self.assertListEqual(
									    lowerCamelCase__					,[
									        SPIECE_UNDERLINE + 'I',
									        SPIECE_UNDERLINE + 'was',
									        SPIECE_UNDERLINE + 'b',
									        'or',
									        'n',
									        SPIECE_UNDERLINE + 'in',
									        SPIECE_UNDERLINE + '',
									        '[UNK]',
									        '2',
									        '0',
									        '0',
									        '0',
									        ',',
									        SPIECE_UNDERLINE + 'and',
									        SPIECE_UNDERLINE + 'this',
									        SPIECE_UNDERLINE + 'is',
									        SPIECE_UNDERLINE + 'f',
									        'al',
									        's',
									        '[UNK]',
									        '.',
									    ]					,)







					@cached_property
					def 	__lowerCAmelCase		(       self     :			Dict    ):
									return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'    )







					@slow
					def 	__lowerCAmelCase		(       self     :			Optional[Any]    ):
									UpperCAmelCase__    =							'Hello World!'
									UpperCAmelCase__    =							[35_389, 6_672, 49, 2]
									self.assertListEqual(lowerCamelCase__					,self.big_tokenizer.encode(lowerCamelCase__    )    )







					@slow
					def 	__lowerCAmelCase		(       self     :			List[str]    ):
									# fmt: off
									UpperCAmelCase__    =							{'input_ids': [[11_073, 82_783, 18, 26, 82_783, 549, 51_540, 248, 17_209, 1_301, 217, 20, 215_186, 1_325, 147, 17_209, 1_301, 217, 20, 56_370, 53, 122_020, 20, 16_477, 27, 87_355, 4_548, 20, 4_728, 78_392, 17, 159_969, 18, 26, 24_491, 629, 15, 538, 22_704, 5_439, 15, 2_788, 24_491, 9_885, 15, 43_534, 605, 15, 814, 18_403, 33_200, 29, 15, 43_534, 24_458, 12_410, 111, 24_966, 83_669, 9_637, 144_068, 26, 850, 22_346, 27, 147, 24_966, 83_669, 83_490, 26, 39_113, 735, 27, 689, 656, 2_800, 1_339, 4_600, 53, 122_020, 115_785, 34, 816, 1_339, 46_887, 18, 147, 53_905, 1_951, 42_238, 41_170, 17_732, 834, 436, 15, 27_523, 98_733, 217, 147, 5_542, 4_981, 930, 17_347, 16, 2], [20_091, 629, 94, 82_786, 58, 490, 20, 1_528, 84, 53_905, 344, 80_592, 110_128, 18_822, 5_267, 1_306, 62, 152_537, 308, 7_997, 401, 124_427, 549, 35_442, 225, 109, 15_055, 25_748, 147, 7_119, 43_712, 34, 767, 135_366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63_784, 119_466, 17, 147_808, 88_214, 18, 656, 81, 32, 3_296, 10_280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=lowerCamelCase__					,model_name='microsoft/xprophetnet-large-wiki100-cased'					,revision='1acad1643ddd54a44df6a1b797ada8373685d90e'					,)



