


import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


snake_case							: int					    =   get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
						from transformers.models.plbart.modeling_plbart import shift_tokens_right

snake_case							: Optional[int]					    =   5_00_03
snake_case							: Optional[int]					    =   5_00_02







@require_sentencepiece
@require_tokenizers
class  snake_case_     (_snake_case			,		unittest.TestCase   ):
					UpperCAmelCase__      :					Dict					   =    PLBartTokenizer
					UpperCAmelCase__      :					Union[str, Any]					   =    None
					UpperCAmelCase__      :					Union[str, Any]					   =    False
					def 				lowerCamelCase__(   self     :str   )      ->					List[Any]:
												super().setUp()

												# We have a SentencePiece fixture for testing
												a__    =  PLBartTokenizer(__snake_case    ,language_codes='base'    ,keep_accents=__snake_case   )
												tokenizer.save_pretrained(self.tmpdirname   )
					def 				lowerCamelCase__(   self     :Union[str, Any]   )      ->					Optional[Any]:
												a__    =  PLBartTokenizer(__snake_case    ,language_codes='base'    ,keep_accents=__snake_case   )

												a__    =  tokenizer.tokenize('This is a test'   )
												self.assertListEqual(__snake_case    ,['▁This', '▁is', '▁a', '▁t', 'est']   )

												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(__snake_case   )    ,[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]    ,)

												a__    =  tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
												self.assertListEqual(
												    __snake_case    ,[
												        SPIECE_UNDERLINE + 'I',
												        SPIECE_UNDERLINE + 'was',
												        SPIECE_UNDERLINE + 'b',
												        'or',
												        'n',
												        SPIECE_UNDERLINE + 'in',
												        SPIECE_UNDERLINE + '',
												        '9',
												        '2',
												        '0',
												        '0',
												        '0',
												        ',',
												        SPIECE_UNDERLINE + 'and',
												        SPIECE_UNDERLINE + 'this',
												        SPIECE_UNDERLINE + 'is',
												        SPIECE_UNDERLINE + 'f',
												        'al',
												        's',
												        'é',
												        '.',
												    ]    ,)
												a__    =  tokenizer.convert_tokens_to_ids(__snake_case   )
												self.assertListEqual(
												    __snake_case    ,[
												        value + tokenizer.fairseq_offset
												        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
												    ]    ,)

												a__    =  tokenizer.convert_ids_to_tokens(__snake_case   )
												self.assertListEqual(
												    __snake_case    ,[
												        SPIECE_UNDERLINE + 'I',
												        SPIECE_UNDERLINE + 'was',
												        SPIECE_UNDERLINE + 'b',
												        'or',
												        'n',
												        SPIECE_UNDERLINE + 'in',
												        SPIECE_UNDERLINE + '',
												        '<unk>',
												        '2',
												        '0',
												        '0',
												        '0',
												        ',',
												        SPIECE_UNDERLINE + 'and',
												        SPIECE_UNDERLINE + 'this',
												        SPIECE_UNDERLINE + 'is',
												        SPIECE_UNDERLINE + 'f',
												        'al',
												        's',
												        '<unk>',
												        '.',
												    ]    ,)

												a__    =  tokenizer.vocab_size
												a__    =  [tokenizer.convert_ids_to_tokens(__snake_case   ) for x in range(end - 4    ,__snake_case   )]

												self.assertListEqual(__snake_case    ,['__java__', '__python__', '__en_XX__', '<mask>']   )

												a__    =  'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
												a__    =  tokenizer(__snake_case   ).input_ids
												self.assertEqual(
												    tokenizer.decode(__snake_case    ,skip_special_tokens=__snake_case    ,clean_up_tokenization_spaces=__snake_case   )    ,__snake_case    ,)



					def 				lowerCamelCase__(   self     :Optional[int]   )      ->					Optional[int]:
												a__    =  PLBartTokenizer(__snake_case    ,language_codes='multi'    ,keep_accents=__snake_case   )

												a__    =  tokenizer.tokenize('This is a test'   )
												self.assertListEqual(__snake_case    ,['▁This', '▁is', '▁a', '▁t', 'est']   )

												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(__snake_case   )    ,[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]    ,)

												a__    =  tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
												self.assertListEqual(
												    __snake_case    ,[
												        SPIECE_UNDERLINE + 'I',
												        SPIECE_UNDERLINE + 'was',
												        SPIECE_UNDERLINE + 'b',
												        'or',
												        'n',
												        SPIECE_UNDERLINE + 'in',
												        SPIECE_UNDERLINE + '',
												        '9',
												        '2',
												        '0',
												        '0',
												        '0',
												        ',',
												        SPIECE_UNDERLINE + 'and',
												        SPIECE_UNDERLINE + 'this',
												        SPIECE_UNDERLINE + 'is',
												        SPIECE_UNDERLINE + 'f',
												        'al',
												        's',
												        'é',
												        '.',
												    ]    ,)
												a__    =  tokenizer.convert_tokens_to_ids(__snake_case   )
												self.assertListEqual(
												    __snake_case    ,[
												        value + tokenizer.fairseq_offset
												        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
												    ]    ,)

												a__    =  tokenizer.convert_ids_to_tokens(__snake_case   )
												self.assertListEqual(
												    __snake_case    ,[
												        SPIECE_UNDERLINE + 'I',
												        SPIECE_UNDERLINE + 'was',
												        SPIECE_UNDERLINE + 'b',
												        'or',
												        'n',
												        SPIECE_UNDERLINE + 'in',
												        SPIECE_UNDERLINE + '',
												        '<unk>',
												        '2',
												        '0',
												        '0',
												        '0',
												        ',',
												        SPIECE_UNDERLINE + 'and',
												        SPIECE_UNDERLINE + 'this',
												        SPIECE_UNDERLINE + 'is',
												        SPIECE_UNDERLINE + 'f',
												        'al',
												        's',
												        '<unk>',
												        '.',
												    ]    ,)
												a__    =  tokenizer.vocab_size
												a__    =  [tokenizer.convert_ids_to_tokens(__snake_case   ) for x in range(end - 7    ,__snake_case   )]

												self.assertListEqual(
												    __snake_case    ,['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__']   )
												a__    =  'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
												a__    =  tokenizer(__snake_case   ).input_ids
												self.assertEqual(
												    tokenizer.decode(__snake_case    ,skip_special_tokens=__snake_case    ,clean_up_tokenization_spaces=__snake_case   )    ,__snake_case    ,)







@require_torch
@require_sentencepiece
@require_tokenizers
class  snake_case_     (unittest.TestCase   ):
					UpperCAmelCase__      :					Optional[int]					   =    'uclanlp/plbart-python-en_XX'
					UpperCAmelCase__      :					Dict					   =    [
					    'def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])',
					    'def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])',
					]
					UpperCAmelCase__      :					Tuple					   =    [
					    'Returns the maximum value of a b c.',
					    'Sums the values of a b c.',
					]
					UpperCAmelCase__      :					str					   =    [
					    1_3_4,
					    5_4_5_2,
					    3_3_4_6_0,
					    3_3_4_4_1,
					    3_3_4_6_3,
					    3_3_4_6_5,
					    3_3_4_6_3,
					    3_3_4_4_9,
					    9_8_8,
					    2_0,
					    3_3_4_5_6,
					    1_9,
					    3_3_4_5_6,
					    7_7_1,
					    3_9,
					    4_2_5_8,
					    8_8_9,
					    3_3_1_8,
					    3_3_4_4_1,
					    3_3_4_6_3,
					    3_3_4_6_5,
					    3_3_4_6_3,
					    3_3_4_4_9,
					    2_4_7_1,
					    2,
					    PYTHON_CODE,
					]
					@classmethod
					def 				lowerCamelCase__(   cls     :Optional[int]   )      ->					List[str]:
												a__    =  PLBartTokenizer.from_pretrained(
												    cls.checkpoint_name    ,language_codes='base'    ,src_lang='python'    ,tgt_lang='en_XX'   )
												a__    =  1
												return cls
					def 				lowerCamelCase__(   self     :Optional[int]   )      ->					List[Any]:
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']    ,5_00_01   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']    ,5_00_02   )
												self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']    ,5_00_03   )
					def 				lowerCamelCase__(   self     :Dict   )      ->					Any:
												a__    =  self.tokenizer.batch_encode_plus(self.src_text   ).input_ids[0]
												self.assertListEqual(self.expected_src_tokens    ,__snake_case   )
					def 				lowerCamelCase__(   self     :Dict   )      ->					Union[str, Any]:
												self.assertIn(__snake_case    ,self.tokenizer.all_special_ids   )
												a__    =  [EN_CODE, 90_37, 3_34_42, 57, 7_52, 1_53, 14, 56, 18, 9, 2]
												a__    =  self.tokenizer.decode(__snake_case    ,skip_special_tokens=__snake_case   )
												a__    =  self.tokenizer.decode(generated_ids[1:]    ,skip_special_tokens=__snake_case   )
												self.assertEqual(__snake_case    ,__snake_case   )
												self.assertNotIn(self.tokenizer.eos_token    ,__snake_case   )
					def 				lowerCamelCase__(   self     :Any   )      ->					str:
												a__    =  ['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 20]
												self.assertIsInstance(src_text[0]    ,__snake_case   )
												a__    =  10
												a__    =  self.tokenizer(__snake_case    ,max_length=__snake_case    ,truncation=__snake_case   ).input_ids[0]
												self.assertEqual(ids[-2]    ,2   )
												self.assertEqual(ids[-1]    ,__snake_case   )
												self.assertEqual(len(__snake_case   )    ,__snake_case   )
					def 				lowerCamelCase__(   self     :Any   )      ->					List[str]:
												self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__']   )    ,[5_00_04, 5_00_01]   )
					def 				lowerCamelCase__(   self     :Union[str, Any]   )      ->					int:
												a__    =  tempfile.mkdtemp()
												a__    =  self.tokenizer.fairseq_tokens_to_ids
												self.tokenizer.save_pretrained(__snake_case   )
												a__    =  PLBartTokenizer.from_pretrained(__snake_case   )
												self.assertDictEqual(new_tok.fairseq_tokens_to_ids    ,__snake_case   )
					@require_torch
					def 				lowerCamelCase__(   self     :Optional[Any]   )      ->					str:
												a__    =  self.tokenizer(self.src_text    ,text_target=self.tgt_text    ,padding=__snake_case    ,return_tensors='pt'   )
												a__    =  shift_tokens_right(batch['labels']    ,self.tokenizer.pad_token_id   )

												# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
												self.assertEqual(batch.input_ids[1][-2:].tolist()    ,[2, PYTHON_CODE]   )
												self.assertEqual(batch.decoder_input_ids[1][0]    ,__snake_case   )
												self.assertEqual(batch.decoder_input_ids[1][-1]    ,2   )
												self.assertEqual(batch.labels[1][-2:].tolist()    ,[2, EN_CODE]   )
					@require_torch
					def 				lowerCamelCase__(   self     :Tuple   )      ->					Union[str, Any]:
												a__    =  self.tokenizer(
												    self.src_text    ,text_target=self.tgt_text    ,padding=__snake_case    ,truncation=__snake_case    ,max_length=len(self.expected_src_tokens   )    ,return_tensors='pt'    ,)
												a__    =  shift_tokens_right(batch['labels']    ,self.tokenizer.pad_token_id   )

												self.assertIsInstance(__snake_case    ,__snake_case   )

												self.assertEqual((2, 26)    ,batch.input_ids.shape   )
												self.assertEqual((2, 26)    ,batch.attention_mask.shape   )
												a__    =  batch.input_ids.tolist()[0]
												self.assertListEqual(self.expected_src_tokens    ,__snake_case   )
												self.assertEqual(2    ,batch.decoder_input_ids[0, -1]   )  # EOS
												# Test that special tokens are reset
												self.assertEqual(self.tokenizer.prefix_tokens    ,[]   )
												self.assertEqual(self.tokenizer.suffix_tokens    ,[self.tokenizer.eos_token_id, PYTHON_CODE]   )
					def 				lowerCamelCase__(   self     :str   )      ->					Dict:
												a__    =  self.tokenizer(self.src_text    ,padding=__snake_case    ,truncation=__snake_case    ,max_length=3    ,return_tensors='pt'   )
												a__    =  self.tokenizer(
												    text_target=self.tgt_text    ,padding=__snake_case    ,truncation=__snake_case    ,max_length=10    ,return_tensors='pt'   )
												a__    =  targets['input_ids']
												a__    =  shift_tokens_right(__snake_case    ,self.tokenizer.pad_token_id   )

												self.assertEqual(batch.input_ids.shape[1]    ,3   )
												self.assertEqual(batch.decoder_input_ids.shape[1]    ,10   )



					@require_torch
					def 				lowerCamelCase__(   self     :Any   )      ->					int:
												a__    =  self.tokenizer._build_translation_inputs(
												    'A test'    ,return_tensors='pt'    ,src_lang='en_XX'    ,tgt_lang='java'   )

												self.assertEqual(
												    nested_simplify(__snake_case   )    ,{
												        # A, test, EOS, en_XX
												        'input_ids': [[1_50, 2_42, 2, 5_00_03]],
												        'attention_mask': [[1, 1, 1, 1]],
												        # java
												        'forced_bos_token_id': 5_00_01,
												    }    ,)








from manim import *







class  snake_case_     (lowerCamelCase_   ):
					def 				lowerCamelCase__(   self     :Optional[Any]   )      ->					Optional[int]:
												a__    =  Rectangle(height=0.5    ,width=0.5   )
												a__    =  Rectangle(height=0.46    ,width=0.46   ).set_stroke(width=0   )

												a__    =  Rectangle(height=0.25    ,width=0.25   )

												a__    =  [mem.copy() for i in range(6   )]
												a__    =  [mem.copy() for i in range(6   )]
												a__    =  VGroup(*__snake_case   ).arrange(__snake_case    ,buff=0   )
												a__    =  VGroup(*__snake_case   ).arrange(__snake_case    ,buff=0   )
												a__    =  VGroup(__snake_case    ,__snake_case   ).arrange(__snake_case    ,buff=0   )
												a__    =  Text('CPU'    ,font_size=24   )
												a__    =  Group(__snake_case    ,__snake_case   ).arrange(__snake_case    ,buff=0.5    ,aligned_edge=__snake_case   )
												cpu.move_to([-2.5, -0.5, 0]   )
												self.add(__snake_case   )

												a__    =  [mem.copy() for i in range(4   )]
												a__    =  VGroup(*__snake_case   ).arrange(__snake_case    ,buff=0   )
												a__    =  Text('GPU'    ,font_size=24   )
												a__    =  Group(__snake_case    ,__snake_case   ).arrange(__snake_case    ,buff=0.5    ,aligned_edge=__snake_case   )
												gpu.move_to([-1, -1, 0]   )
												self.add(__snake_case   )

												a__    =  [mem.copy() for i in range(6   )]
												a__    =  VGroup(*__snake_case   ).arrange(__snake_case    ,buff=0   )

												a__    =  Text('Model'    ,font_size=24   )
												a__    =  Group(__snake_case    ,__snake_case   ).arrange(__snake_case    ,buff=0.5    ,aligned_edge=__snake_case   )
												model.move_to([3, -1.0, 0]   )
												self.add(__snake_case   )

												a__    =  []
												a__    =  []

												for i, rect in enumerate(__snake_case   ):
																			a__    =  fill.copy().set_fill(__snake_case    ,opacity=0.8   )
																			target.move_to(__snake_case   )
																			model_arr.append(__snake_case   )

																			a__    =  Rectangle(height=0.46    ,width=0.46   ).set_stroke(width=0.0   ).set_fill(__snake_case    ,opacity=0.8   )
																			cpu_target.move_to(cpu_left_col_base[i]   )
																			model_cpu_arr.append(__snake_case   )

												self.add(*__snake_case    ,*__snake_case   )

												a__    =  [meta_mem.copy() for i in range(6   )]
												a__    =  [meta_mem.copy() for i in range(6   )]
												a__    =  VGroup(*__snake_case   ).arrange(__snake_case    ,buff=0   )
												a__    =  VGroup(*__snake_case   ).arrange(__snake_case    ,buff=0   )
												a__    =  VGroup(__snake_case    ,__snake_case   ).arrange(__snake_case    ,buff=0   )
												a__    =  Text('Disk'    ,font_size=24   )
												a__    =  Group(__snake_case    ,__snake_case   ).arrange(__snake_case    ,buff=0.5    ,aligned_edge=__snake_case   )
												disk.move_to([-4, -1.25, 0]   )
												self.add(__snake_case    ,__snake_case   )

												a__    =  Square(side_length=2.2   )
												key.move_to([-5, 2, 0]   )

												a__    =  MarkupText(
												    F'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'    ,font_size=18    ,)

												key_text.move_to([-5, 2.4, 0]   )

												self.add(__snake_case    ,__snake_case   )

												a__    =  MarkupText(
												    F'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'    ,font_size=18    ,)

												blue_text.next_to(__snake_case    ,DOWN * 2.4    ,aligned_edge=key_text.get_left()   )
												self.add(__snake_case   )

												a__    =  MarkupText(
												    F'Now watch as an input is passed through the model\nand how the memory is utilized and handled.'    ,font_size=24    ,)
												step_a.move_to([2, 2, 0]   )

												self.play(Write(__snake_case   )   )

												a__    =  Square(0.3   )
												input.set_fill(__snake_case    ,opacity=1.0   )
												input.set_stroke(width=0.0   )
												input.next_to(model_base[0]    ,__snake_case    ,buff=0.5   )

												self.play(Write(__snake_case   )   )

												input.generate_target()
												input.target.next_to(model_arr[0]    ,direction=__snake_case    ,buff=0.02   )
												self.play(MoveToTarget(__snake_case   )   )

												self.play(FadeOut(__snake_case   )   )

												a__    =  Arrow(start=__snake_case    ,end=__snake_case    ,color=__snake_case    ,buff=0.5   )
												a.next_to(model_arr[0].get_left()    ,__snake_case    ,buff=0.2   )

												model_cpu_arr[0].generate_target()
												model_cpu_arr[0].target.move_to(gpu_rect[0]   )

												a__    =  MarkupText(
												    F'As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back.'    ,font_size=24    ,)
												step_a.move_to([2, 2, 0]   )

												self.play(Write(__snake_case    ,run_time=3   )   )

												a__    =  {'run_time': 1, 'fade_in': True, 'fade_out': True, 'buff': 0.02}

												self.play(
												    Write(__snake_case   )    ,Circumscribe(model_arr[0]    ,color=__snake_case    ,**__snake_case   )    ,Circumscribe(model_cpu_arr[0]    ,color=__snake_case    ,**__snake_case   )    ,Circumscribe(gpu_rect[0]    ,color=__snake_case    ,**__snake_case   )    ,)
												self.play(MoveToTarget(model_cpu_arr[0]   )   )

												a__    =  a.copy()
												for i in range(6   ):
																			a_c.next_to(model_arr[i].get_right() + 0.02    ,__snake_case    ,buff=0.2   )

																			input.generate_target()
																			input.target.move_to(model_arr[i].get_right() + 0.02   )

																			a__    =  AnimationGroup(
																			    FadeOut(__snake_case    ,run_time=0.5   )    ,MoveToTarget(__snake_case    ,run_time=0.5   )    ,FadeIn(__snake_case    ,run_time=0.5   )    ,lag_ratio=0.2   )

																			self.play(__snake_case   )

																			model_cpu_arr[i].generate_target()
																			model_cpu_arr[i].target.move_to(cpu_left_col_base[i]   )

																			if i < 5:
																										model_cpu_arr[i + 1].generate_target()
																										model_cpu_arr[i + 1].target.move_to(gpu_rect[0]   )
																										if i >= 1:
																																	a__    =  0.7

																										self.play(
																										    Circumscribe(model_arr[i]    ,**__snake_case   )    ,Circumscribe(cpu_left_col_base[i]    ,**__snake_case   )    ,Circumscribe(cpu_left_col_base[i + 1]    ,color=__snake_case    ,**__snake_case   )    ,Circumscribe(gpu_rect[0]    ,color=__snake_case    ,**__snake_case   )    ,Circumscribe(model_arr[i + 1]    ,color=__snake_case    ,**__snake_case   )    ,)
																										if i < 1:
																																	self.play(
																																	    MoveToTarget(model_cpu_arr[i]   )    ,MoveToTarget(model_cpu_arr[i + 1]   )    ,)
																										else:
																																	self.play(
																																	    MoveToTarget(model_cpu_arr[i]    ,run_time=0.7   )    ,MoveToTarget(model_cpu_arr[i + 1]    ,run_time=0.7   )    ,)
																			else:
																										model_cpu_arr[i].generate_target()
																										model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]   )
																										input.generate_target()
																										input.target.next_to(model_arr[-1].get_right()    ,RIGHT + 0.02    ,buff=0.2   )

																										self.play(
																										    Circumscribe(model_arr[-1]    ,color=__snake_case    ,**__snake_case   )    ,Circumscribe(cpu_left_col_base[-1]    ,color=__snake_case    ,**__snake_case   )    ,Circumscribe(gpu_rect[0]    ,color=__snake_case    ,**__snake_case   )    ,)

																										self.play(MoveToTarget(model_cpu_arr[i]   )   )

																			a__    =  a_c
																			a__    =  a_c.copy()

												input.generate_target()
												input.target.next_to(model_base[-1]    ,RIGHT + 0.02    ,buff=0.5   )
												self.play(
												    FadeOut(__snake_case   )    ,FadeOut(__snake_case    ,run_time=0.5   )    ,)

												a__    =  MarkupText(F'Inference on a model too large for GPU memory\nis successfully completed.'    ,font_size=24   )
												step_a.move_to([2, 2, 0]   )

												self.play(Write(__snake_case    ,run_time=3   )    ,MoveToTarget(__snake_case   )   )

												self.wait()




