
import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin







class        lowercase	(      __A ,						unittest.TestCase	):
							lowercase_ : Optional[Any]       =ProphetNetTokenizer
							lowercase_ : List[str]       =False




							def 				A__   (       self):
												super().setUp()

												lowercase					=		[
												    '[UNK]',
												    '[CLS]',
												    '[SEP]',
												    '[PAD]',
												    '[MASK]',
												    'want',
												    '##want',
												    '##ed',
												    'wa',
												    'un',
												    'runn',
												    '##ing',
												    ',',
												    'low',
												    'lowest',
												]
												lowercase					=		os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES['''vocab_file'''])
												with open(self.vocab_file     ,'''w'''     ,encoding='''utf-8''') as vocab_writer:
																	vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))




							def 				A__   (       self     ,A__):
												lowercase					=		'UNwant\u00E9d,running'
												lowercase					=		'unwanted, running'
												return input_text, output_text




							def 				A__   (       self):
												lowercase					=		self.tokenizer_class(self.vocab_file)

												lowercase					=		tokenizer.tokenize('''UNwant\u00E9d,running''')
												self.assertListEqual(A__     ,['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''])
												self.assertListEqual(tokenizer.convert_tokens_to_ids(A__)     ,[9, 6, 7, 1_2, 1_0, 1_1])




							def 				A__   (       self):
												lowercase					=		BasicTokenizer()

												self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''')     ,['''ah''', '''\u535A''', '''\u63A8''', '''zz'''])




							def 				A__   (       self):
												lowercase					=		BasicTokenizer(do_lower_case=A__)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')     ,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''])
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')     ,['''hello'''])




							def 				A__   (       self):
												lowercase					=		BasicTokenizer(do_lower_case=A__     ,strip_accents=A__)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')     ,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')     ,['''h\u00E9llo'''])




							def 				A__   (       self):
												lowercase					=		BasicTokenizer(do_lower_case=A__     ,strip_accents=A__)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')     ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')     ,['''hello'''])




							def 				A__   (       self):
												lowercase					=		BasicTokenizer(do_lower_case=A__)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')     ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
												self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')     ,['''hello'''])




							def 				A__   (       self):
												lowercase					=		BasicTokenizer(do_lower_case=A__)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')     ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])




							def 				A__   (       self):
												lowercase					=		BasicTokenizer(do_lower_case=A__     ,strip_accents=A__)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')     ,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])




							def 				A__   (       self):
												lowercase					=		BasicTokenizer(do_lower_case=A__     ,strip_accents=A__)

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')     ,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])




							def 				A__   (       self):
												lowercase					=		BasicTokenizer(do_lower_case=A__     ,never_split=['''[UNK]'''])

												self.assertListEqual(
												    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''')     ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''])




							def 				A__   (       self):
												lowercase					=		['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

												lowercase					=		{}
												for i, token in enumerate(A__):
																	lowercase					=		i
												lowercase					=		WordpieceTokenizer(vocab=A__     ,unk_token='''[UNK]''')

												self.assertListEqual(tokenizer.tokenize('''''')     ,[])

												self.assertListEqual(tokenizer.tokenize('''unwanted running''')     ,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''])

												self.assertListEqual(tokenizer.tokenize('''unwantedX running''')     ,['''[UNK]''', '''runn''', '''##ing'''])




							@require_torch
							def 				A__   (       self):
												lowercase					=		self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased''')

												lowercase					=		['A long paragraph for summarization.', 'Another paragraph for summarization.']
												lowercase					=		[1_0_3_7, 2_1_4_6, 2_0_4_2_3, 2_0_0_5, 7_6_8_0, 7_8_4_9, 3_9_8_9, 1_0_1_2, 1_0_2]
												lowercase					=		tokenizer(A__     ,padding=A__     ,return_tensors='''pt''')
												self.assertIsInstance(A__     ,A__)
												lowercase					=		list(batch.input_ids.numpy()[0])
												self.assertListEqual(A__     ,A__)

												self.assertEqual((2, 9)     ,batch.input_ids.shape)
												self.assertEqual((2, 9)     ,batch.attention_mask.shape)




							def 				A__   (       self):
												self.assertTrue(_is_whitespace(''' '''))
												self.assertTrue(_is_whitespace('''\t'''))
												self.assertTrue(_is_whitespace('''\r'''))
												self.assertTrue(_is_whitespace('''\n'''))
												self.assertTrue(_is_whitespace('''\u00A0'''))

												self.assertFalse(_is_whitespace('''A'''))
												self.assertFalse(_is_whitespace('''-'''))




							def 				A__   (       self):
												self.assertTrue(_is_control('''\u0005'''))

												self.assertFalse(_is_control('''A'''))
												self.assertFalse(_is_control(''' '''))
												self.assertFalse(_is_control('''\t'''))
												self.assertFalse(_is_control('''\r'''))




							def 				A__   (       self):
												self.assertTrue(_is_punctuation('''-'''))
												self.assertTrue(_is_punctuation('''$'''))
												self.assertTrue(_is_punctuation('''`'''))
												self.assertTrue(_is_punctuation('''.'''))

												self.assertFalse(_is_punctuation('''A'''))
												self.assertFalse(_is_punctuation(''' '''))





							@slow
							def 				A__   (       self):
												lowercase					=		self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased''')

												lowercase					=		tokenizer.encode('''sequence builders'''     ,add_special_tokens=A__)
												lowercase					=		tokenizer.encode('''multi-sequence build'''     ,add_special_tokens=A__)

												lowercase					=		tokenizer.build_inputs_with_special_tokens(A__)
												lowercase					=		tokenizer.build_inputs_with_special_tokens(A__     ,A__)

												assert encoded_sentence == text + [1_0_2]
												assert encoded_pair == text + [1_0_2] + text_a + [1_0_2]






import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
							import os

							import jax.numpy as jnp
							from jax import jit

							from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
							from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

							lowercase__	:Optional[Any]										=					"0.12"  # assumed parallelism: 8


if is_torch_available():
							import torch



def  UpperCamelCase					(							lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__=None	):
					'''simple docstring'''
					if rng is None:
										lowercase					=		random.Random()

					lowercase					=		1
					for dim in shape:
										total_dims *= dim

					lowercase					=		[]
					for _ in range(lowerCAmelCase__	):
										values.append(rng.randint(0				,    vocab_size - 1	)	)

					lowercase					=		np.array(lowerCAmelCase__				,    dtype=jnp.intaa	).reshape(lowerCAmelCase__	)

					return output



def  UpperCamelCase					(							lowerCAmelCase__				,    lowerCAmelCase__=None	):
					'''simple docstring'''
					lowercase					=		ids_tensor(lowerCAmelCase__				,    vocab_size=2				,    rng=lowerCAmelCase__	)
					# make sure that at least one token is attended to for each batch
					lowercase					=		1
					return attn_mask







@require_flax
class        lowercase	:
							lowercase_ : Any       =None
							lowercase_ : List[str]       =()




							def 				A__   (       self):
												lowercase			,		lowercase					=		self.model_tester.prepare_config_and_inputs_for_common()

												# cut to half length & take max batch_size 3
												lowercase					=		2
												lowercase					=		inputs['''input_ids'''].shape[-1] // 2
												lowercase					=		inputs['''input_ids'''][:max_batch_size, :sequence_length]

												lowercase					=		jnp.ones_like(A__)
												lowercase					=		attention_mask[:max_batch_size, :sequence_length]

												# generate max 5 tokens
												lowercase					=		input_ids.shape[-1] + 5
												if config.eos_token_id is not None and config.pad_token_id is None:
																	# hack to allow generate for models such as GPT2 as is done in `generate()`
																	lowercase					=		config.eos_token_id
												return config, input_ids, attention_mask, max_length




							@is_pt_flax_cross_test
							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()
												lowercase					=		False
												lowercase					=		max_length
												lowercase					=		0

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model_class.__name__[4:]  # Skip the "Flax" at the beginning
																	lowercase					=		getattr(A__     ,A__)
																	lowercase					=		pt_model_class(A__).eval()
																	lowercase					=		load_flax_weights_in_pytorch_model(A__     ,flax_model.params)

																	lowercase					=		flax_model.generate(A__).sequences
																	lowercase					=		pt_model.generate(torch.tensor(A__     ,dtype=torch.long))

																	if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
																						lowercase					=		flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

																	self.assertListEqual(pt_generation_outputs.numpy().tolist()     ,flax_generation_outputs.tolist())




							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()
												lowercase					=		False
												lowercase					=		max_length

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model.generate(A__).sequences
																	self.assertEqual(generation_outputs.shape[-1]     ,A__)

																	lowercase					=		jit(model.generate)
																	lowercase					=		jit_generate(A__).sequences

																	self.assertListEqual(generation_outputs.tolist()     ,jit_generation_outputs.tolist())




							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()
												lowercase					=		True
												lowercase					=		max_length

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model.generate(A__).sequences
																	self.assertEqual(generation_outputs.shape[-1]     ,A__)

																	lowercase					=		jit(model.generate)
																	lowercase					=		jit_generate(A__).sequences

																	self.assertListEqual(generation_outputs.tolist()     ,jit_generation_outputs.tolist())




							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()
												lowercase					=		False
												lowercase					=		max_length
												lowercase					=		2

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model.generate(A__).sequences
																	self.assertEqual(generation_outputs.shape[-1]     ,A__)

																	lowercase					=		jit(model.generate)
																	lowercase					=		jit_generate(A__).sequences

																	self.assertListEqual(generation_outputs.tolist()     ,jit_generation_outputs.tolist())




							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()
												lowercase					=		False
												lowercase					=		max_length
												lowercase					=		2
												lowercase					=		2

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model.generate(A__).sequences
																	self.assertEqual(generation_outputs.shape[0]     ,input_ids.shape[0] * config.num_return_sequences)




							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()
												lowercase					=		True
												lowercase					=		max_length
												lowercase					=		0.8
												lowercase					=		1_0
												lowercase					=		0.3
												lowercase					=		1
												lowercase					=		8
												lowercase					=		9

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model.generate(A__).sequences
																	self.assertEqual(generation_outputs.shape[-1]     ,A__)

																	lowercase					=		jit(model.generate)
																	lowercase					=		jit_generate(A__).sequences

																	self.assertListEqual(generation_outputs.tolist()     ,jit_generation_outputs.tolist())




							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()
												lowercase					=		max_length
												lowercase					=		1
												lowercase					=		8
												lowercase					=		9

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model.generate(A__).sequences
																	self.assertEqual(generation_outputs.shape[-1]     ,A__)

																	lowercase					=		jit(model.generate)
																	lowercase					=		jit_generate(A__).sequences

																	self.assertListEqual(generation_outputs.tolist()     ,jit_generation_outputs.tolist())




							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()
												lowercase					=		max_length
												lowercase					=		2
												lowercase					=		1
												lowercase					=		8
												lowercase					=		9

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model.generate(A__).sequences
																	self.assertEqual(generation_outputs.shape[-1]     ,A__)

																	lowercase					=		jit(model.generate)
																	lowercase					=		jit_generate(A__).sequences

																	self.assertListEqual(generation_outputs.tolist()     ,jit_generation_outputs.tolist())




							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()

												# pad attention mask on the left
												lowercase					=		attention_mask.at[(0, 0)].set(0)

												lowercase					=		False
												lowercase					=		max_length

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model.generate(A__     ,attention_mask=A__).sequences
																	self.assertEqual(generation_outputs.shape[-1]     ,A__)

																	lowercase					=		jit(model.generate)
																	lowercase					=		jit_generate(A__     ,attention_mask=A__).sequences

																	self.assertListEqual(generation_outputs.tolist()     ,jit_generation_outputs.tolist())




							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()

												# pad attention mask on the left
												lowercase					=		attention_mask.at[(0, 0)].set(0)

												lowercase					=		True
												lowercase					=		max_length

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model.generate(A__     ,attention_mask=A__).sequences
																	self.assertEqual(generation_outputs.shape[-1]     ,A__)

																	lowercase					=		jit(model.generate)
																	lowercase					=		jit_generate(A__     ,attention_mask=A__).sequences

																	self.assertListEqual(generation_outputs.tolist()     ,jit_generation_outputs.tolist())





							def 				A__   (       self):
												lowercase			,		lowercase			,		lowercase			,		lowercase					=		self._get_input_ids_and_config()

												# pad attention mask on the left
												lowercase					=		attention_mask.at[(0, 0)].set(0)

												lowercase					=		2
												lowercase					=		max_length

												for model_class in self.all_generative_model_classes:
																	lowercase					=		model_class(A__)

																	lowercase					=		model.generate(A__     ,attention_mask=A__).sequences
																	self.assertEqual(generation_outputs.shape[-1]     ,A__)

																	lowercase					=		jit(model.generate)
																	lowercase					=		jit_generate(A__     ,attention_mask=A__).sequences

																	self.assertListEqual(generation_outputs.tolist()     ,jit_generation_outputs.tolist())







@require_flax
class        lowercase	(      unittest.TestCase	):




							def 				A__   (       self):
												lowercase					=		AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-bert''')
												lowercase					=		FlaxAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-bert-flax-only''')

												lowercase					=		'''Hello world'''
												lowercase					=		tokenizer(A__     ,return_tensors='''np''').input_ids

												# typos are quickly detected (the correct argument is `do_sample`)
												with self.assertRaisesRegex(A__     ,'''do_samples'''):
																	model.generate(A__     ,do_samples=A__)

												# arbitrary arguments that will not be used anywhere are also not accepted
												with self.assertRaisesRegex(A__     ,'''foo'''):
																	lowercase					=		{'''foo''': '''bar'''}
																	model.generate(A__     ,**A__)




