



import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__     = get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
							from transformers.models.plbart.modeling_plbart import shift_tokens_right

UpperCAmelCase__     = 5_0003
UpperCAmelCase__     = 5_0002







@require_sentencepiece
@require_tokenizers
class     __lowerCAmelCase					(    A       ,		unittest.TestCase		):
			UpperCamelCase           =  PLBartTokenizer
			UpperCamelCase           =  None
			UpperCamelCase           =  False



			def 					_lowerCamelCase (		self     :						Union[str, Any])     ->					Union[str, Any]:



						"""simple docstring"""






						super().setUp()

						# We have a SentencePiece fixture for testing
						_UpperCAmelCase	 =     PLBartTokenizer(A				,	language_codes='base'				,	keep_accents=A)
						tokenizer.save_pretrained(self.tmpdirname)



			def 					_lowerCamelCase (		self     :						List[Any])     ->					Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase	 =     PLBartTokenizer(A				,	language_codes='base'				,	keep_accents=A)

						_UpperCAmelCase	 =     tokenizer.tokenize('This is a test')
						self.assertListEqual(A				,	['▁This', '▁is', '▁a', '▁t', 'est'])

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(A)				,	[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]				,	)

						_UpperCAmelCase	 =     tokenizer.tokenize('I was born in 92000, and this is falsé.')
						self.assertListEqual(
						    A				,	[
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '9',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        'é',
						        '.',
						    ]				,	)
						_UpperCAmelCase	 =     tokenizer.convert_tokens_to_ids(A)
						self.assertListEqual(
						    A				,	[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
						    ]				,	)

						_UpperCAmelCase	 =     tokenizer.convert_ids_to_tokens(A)
						self.assertListEqual(
						    A				,	[
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '<unk>',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        '<unk>',
						        '.',
						    ]				,	)

						_UpperCAmelCase	 =     tokenizer.vocab_size
						_UpperCAmelCase	 =     [tokenizer.convert_ids_to_tokens(A) for x in range(end - 4				,	A)]

						self.assertListEqual(A				,	['__java__', '__python__', '__en_XX__', '<mask>'])

						_UpperCAmelCase	 =     'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
						_UpperCAmelCase	 =     tokenizer(A).input_ids
						self.assertEqual(
						    tokenizer.decode(A				,	skip_special_tokens=A				,	clean_up_tokenization_spaces=A)				,	A				,	)



			def 					_lowerCamelCase (		self     :						List[Any])     ->					Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase	 =     PLBartTokenizer(A				,	language_codes='multi'				,	keep_accents=A)

						_UpperCAmelCase	 =     tokenizer.tokenize('This is a test')
						self.assertListEqual(A				,	['▁This', '▁is', '▁a', '▁t', 'est'])

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(A)				,	[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]				,	)

						_UpperCAmelCase	 =     tokenizer.tokenize('I was born in 92000, and this is falsé.')
						self.assertListEqual(
						    A				,	[
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '9',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        'é',
						        '.',
						    ]				,	)
						_UpperCAmelCase	 =     tokenizer.convert_tokens_to_ids(A)
						self.assertListEqual(
						    A				,	[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
						    ]				,	)

						_UpperCAmelCase	 =     tokenizer.convert_ids_to_tokens(A)
						self.assertListEqual(
						    A				,	[
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '<unk>',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        '<unk>',
						        '.',
						    ]				,	)
						_UpperCAmelCase	 =     tokenizer.vocab_size
						_UpperCAmelCase	 =     [tokenizer.convert_ids_to_tokens(A) for x in range(end - 7				,	A)]

						self.assertListEqual(
						    A				,	['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__'])
						_UpperCAmelCase	 =     'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
						_UpperCAmelCase	 =     tokenizer(A).input_ids
						self.assertEqual(
						    tokenizer.decode(A				,	skip_special_tokens=A				,	clean_up_tokenization_spaces=A)				,	A				,	)







@require_torch
@require_sentencepiece
@require_tokenizers
class     __lowerCAmelCase					(    unittest.TestCase		):
			UpperCamelCase           =  '''uclanlp/plbart-python-en_XX'''
			UpperCamelCase           =  [
			    '''def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])''',
			    '''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''',
			]
			UpperCamelCase           =  [
			    '''Returns the maximum value of a b c.''',
			    '''Sums the values of a b c.''',
			]
			UpperCamelCase           =  [
			    1_3_4,
			    5_4_5_2,
			    3_3_4_6_0,
			    3_3_4_4_1,
			    3_3_4_6_3,
			    3_3_4_6_5,
			    3_3_4_6_3,
			    3_3_4_4_9,
			    9_8_8,
			    2_0,
			    3_3_4_5_6,
			    1_9,
			    3_3_4_5_6,
			    7_7_1,
			    3_9,
			    4_2_5_8,
			    8_8_9,
			    3_3_1_8,
			    3_3_4_4_1,
			    3_3_4_6_3,
			    3_3_4_6_5,
			    3_3_4_6_3,
			    3_3_4_4_9,
			    2_4_7_1,
			    2,
			    PYTHON_CODE,
			]



			@classmethod
			def 					_lowerCamelCase (		cls     :						List[str])     ->					Optional[Any]:



						"""simple docstring"""






						_UpperCAmelCase	 =     PLBartTokenizer.from_pretrained(
						    cls.checkpoint_name				,	language_codes='base'				,	src_lang='python'				,	tgt_lang='en_XX')
						_UpperCAmelCase	 =     1
						return cls



			def 					_lowerCamelCase (		self     :						List[Any])     ->					int:



						"""simple docstring"""






						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']				,	5_00_01)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']				,	5_00_02)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']				,	5_00_03)



			def 					_lowerCamelCase (		self     :						Tuple)     ->					List[Any]:



						"""simple docstring"""






						_UpperCAmelCase	 =     self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
						self.assertListEqual(self.expected_src_tokens				,	A)



			def 					_lowerCamelCase (		self     :						Dict)     ->					Optional[Any]:



						"""simple docstring"""






						self.assertIn(A				,	self.tokenizer.all_special_ids)
						_UpperCAmelCase	 =     [EN_CODE, 90_37, 3_34_42, 57, 7_52, 1_53, 14, 56, 18, 9, 2]
						_UpperCAmelCase	 =     self.tokenizer.decode(A				,	skip_special_tokens=A)
						_UpperCAmelCase	 =     self.tokenizer.decode(generated_ids[1:]				,	skip_special_tokens=A)
						self.assertEqual(A				,	A)
						self.assertNotIn(self.tokenizer.eos_token				,	A)



			def 					_lowerCamelCase (		self     :						Union[str, Any])     ->					Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase	 =     ['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 20]
						self.assertIsInstance(src_text[0]				,	A)
						_UpperCAmelCase	 =     10
						_UpperCAmelCase	 =     self.tokenizer(A				,	max_length=A				,	truncation=A).input_ids[0]
						self.assertEqual(ids[-2]				,	2)
						self.assertEqual(ids[-1]				,	A)
						self.assertEqual(len(A)				,	A)



			def 					_lowerCamelCase (		self     :						Optional[int])     ->					Tuple:



						"""simple docstring"""






						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__'])				,	[5_00_04, 5_00_01])



			def 					_lowerCamelCase (		self     :						Any)     ->					Optional[int]:



						"""simple docstring"""






						_UpperCAmelCase	 =     tempfile.mkdtemp()
						_UpperCAmelCase	 =     self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(A)
						_UpperCAmelCase	 =     PLBartTokenizer.from_pretrained(A)
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids				,	A)



			@require_torch
			def 					_lowerCamelCase (		self     :						Tuple)     ->					Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase	 =     self.tokenizer(self.src_text				,	text_target=self.tgt_text				,	padding=A				,	return_tensors='pt')
						_UpperCAmelCase	 =     shift_tokens_right(batch['labels']				,	self.tokenizer.pad_token_id)

						# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
						self.assertEqual(batch.input_ids[1][-2:].tolist()				,	[2, PYTHON_CODE])
						self.assertEqual(batch.decoder_input_ids[1][0]				,	A)
						self.assertEqual(batch.decoder_input_ids[1][-1]				,	2)
						self.assertEqual(batch.labels[1][-2:].tolist()				,	[2, EN_CODE])



			@require_torch
			def 					_lowerCamelCase (		self     :						List[str])     ->					Any:



						"""simple docstring"""






						_UpperCAmelCase	 =     self.tokenizer(
						    self.src_text				,	text_target=self.tgt_text				,	padding=A				,	truncation=A				,	max_length=len(self.expected_src_tokens)				,	return_tensors='pt'				,	)
						_UpperCAmelCase	 =     shift_tokens_right(batch['labels']				,	self.tokenizer.pad_token_id)

						self.assertIsInstance(A				,	A)

						self.assertEqual((2, 26)				,	batch.input_ids.shape)
						self.assertEqual((2, 26)				,	batch.attention_mask.shape)
						_UpperCAmelCase	 =     batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens				,	A)
						self.assertEqual(2				,	batch.decoder_input_ids[0, -1])  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens				,	[])
						self.assertEqual(self.tokenizer.suffix_tokens				,	[self.tokenizer.eos_token_id, PYTHON_CODE])



			def 					_lowerCamelCase (		self     :						Union[str, Any])     ->					Optional[Any]:



						"""simple docstring"""






						_UpperCAmelCase	 =     self.tokenizer(self.src_text				,	padding=A				,	truncation=A				,	max_length=3				,	return_tensors='pt')
						_UpperCAmelCase	 =     self.tokenizer(
						    text_target=self.tgt_text				,	padding=A				,	truncation=A				,	max_length=10				,	return_tensors='pt')
						_UpperCAmelCase	 =     targets['input_ids']
						_UpperCAmelCase	 =     shift_tokens_right(A				,	self.tokenizer.pad_token_id)

						self.assertEqual(batch.input_ids.shape[1]				,	3)
						self.assertEqual(batch.decoder_input_ids.shape[1]				,	10)



			@require_torch
			def 					_lowerCamelCase (		self     :						Dict)     ->					Dict:



						"""simple docstring"""






						_UpperCAmelCase	 =     self.tokenizer._build_translation_inputs(
						    'A test'				,	return_tensors='pt'				,	src_lang='en_XX'				,	tgt_lang='java')

						self.assertEqual(
						    nested_simplify(A)				,	{
						        # A, test, EOS, en_XX
						        'input_ids': [[1_50, 2_42, 2, 5_00_03]],
						        'attention_mask': [[1, 1, 1, 1]],
						        # java
						        'forced_bos_token_id': 5_00_01,
						    }				,	)





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCAmelCase__     = {
    "configuration_ctrl": ["CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP", "CTRLConfig"],
    "tokenization_ctrl": ["CTRLTokenizer"],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__     = [
							    "CTRL_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "CTRLForSequenceClassification",
							    "CTRLLMHeadModel",
							    "CTRLModel",
							    "CTRLPreTrainedModel",
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__     = [
							    "TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "TFCTRLForSequenceClassification",
							    "TFCTRLLMHeadModel",
							    "TFCTRLModel",
							    "TFCTRLPreTrainedModel",
							]


if TYPE_CHECKING:
							from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
							from .tokenization_ctrl import CTRLTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_ctrl import (
														    CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
														    CTRLForSequenceClassification,
														    CTRLLMHeadModel,
														    CTRLModel,
														    CTRLPreTrainedModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_ctrl import (
														    TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFCTRLForSequenceClassification,
														    TFCTRLLMHeadModel,
														    TFCTRLModel,
														    TFCTRLPreTrainedModel,
														)

else:
							import sys

							UpperCAmelCase__     = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
