





import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin


class 			lowercase				(							a_		,			unittest.TestCase     ):



						"""simple docstring"""

						_UpperCamelCase  :	Optional[int]								=	ProphetNetTokenizer
						_UpperCamelCase  :	Any								=	False







						def 		__UpperCAmelCase       (    self			:		int					):





										'''simple docstring'''







										super().setUp()

										_snake_case :			Any  						=   [
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										_snake_case :			int  						=   os.path.join(self.tmpdirname ,				VOCAB_FILES_NAMES['vocab_file']					)
										with open(self.vocab_file ,				'w' ,				encoding='utf-8'					) as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)







						def 		__UpperCAmelCase       (    self			:		Optional[Any] ,				lowerCamelCase_			:		Union[str, Any]					):





										'''simple docstring'''







										_snake_case :			Any  						=   'UNwant\u00E9d,running'
										_snake_case :			List[str]  						=   'unwanted, running'
										return input_text, output_text







						def 		__UpperCAmelCase       (    self			:		Any					):





										'''simple docstring'''







										_snake_case :			Optional[int]  						=   self.tokenizer_class(self.vocab_file					)

										_snake_case :			str  						=   tokenizer.tokenize('UNwant\u00E9d,running'					)
										self.assertListEqual(lowerCamelCase_ ,				['un', '##want', '##ed', ',', 'runn', '##ing']					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_					) ,				[9, 6, 7, 12, 10, 11]					)







						def 		__UpperCAmelCase       (    self			:		Dict					):





										'''simple docstring'''







										_snake_case :			Tuple  						=   BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					) ,				['ah', '\u535A', '\u63A8', 'zz']					)







						def 		__UpperCAmelCase       (    self			:		int					):





										'''simple docstring'''







										_snake_case :			Union[str, Any]  						=   BasicTokenizer(do_lower_case=lowerCamelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					) ,				['hello', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					) ,				['hello']					)







						def 		__UpperCAmelCase       (    self			:		Any					):





										'''simple docstring'''







										_snake_case :			int  						=   BasicTokenizer(do_lower_case=lowerCamelCase_ ,				strip_accents=lowerCamelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					) ,				['hällo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					) ,				['h\u00E9llo']					)







						def 		__UpperCAmelCase       (    self			:		List[Any]					):





										'''simple docstring'''







										_snake_case :			int  						=   BasicTokenizer(do_lower_case=lowerCamelCase_ ,				strip_accents=lowerCamelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					) ,				['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					) ,				['hello']					)







						def 		__UpperCAmelCase       (    self			:		List[str]					):





										'''simple docstring'''







										_snake_case :			Optional[Any]  						=   BasicTokenizer(do_lower_case=lowerCamelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					) ,				['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					) ,				['hello']					)







						def 		__UpperCAmelCase       (    self			:		Any					):





										'''simple docstring'''







										_snake_case :			Optional[Any]  						=   BasicTokenizer(do_lower_case=lowerCamelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					) ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)







						def 		__UpperCAmelCase       (    self			:		List[Any]					):





										'''simple docstring'''







										_snake_case :			Union[str, Any]  						=   BasicTokenizer(do_lower_case=lowerCamelCase_ ,				strip_accents=lowerCamelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					) ,				['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)







						def 		__UpperCAmelCase       (    self			:		Tuple					):





										'''simple docstring'''







										_snake_case :			List[str]  						=   BasicTokenizer(do_lower_case=lowerCamelCase_ ,				strip_accents=lowerCamelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					) ,				['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)







						def 		__UpperCAmelCase       (    self			:		Optional[int]					):





										'''simple docstring'''







										_snake_case :			List[Any]  						=   BasicTokenizer(do_lower_case=lowerCamelCase_ ,				never_split=['[UNK]']					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					) ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)







						def 		__UpperCAmelCase       (    self			:		List[Any]					):





										'''simple docstring'''







										_snake_case :			List[str]  						=   ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										_snake_case :			Optional[int]  						=   {}
										for i, token in enumerate(lowerCamelCase_					):
														_snake_case :			Any  						=   i
										_snake_case :			Dict  						=   WordpieceTokenizer(vocab=lowerCamelCase_ ,				unk_token='[UNK]'					)

										self.assertListEqual(tokenizer.tokenize(''					) ,				[]					)

										self.assertListEqual(tokenizer.tokenize('unwanted running'					) ,				['un', '##want', '##ed', 'runn', '##ing']					)

										self.assertListEqual(tokenizer.tokenize('unwantedX running'					) ,				['[UNK]', 'runn', '##ing']					)







						@require_torch
						def 		__UpperCAmelCase       (    self			:		List[str]					):





										'''simple docstring'''







										_snake_case :			List[Any]  						=   self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'					)

										_snake_case :			Union[str, Any]  						=   ['A long paragraph for summarization.', 'Another paragraph for summarization.']
										_snake_case :			List[Any]  						=   [10_37, 21_46, 2_04_23, 20_05, 76_80, 78_49, 39_89, 10_12, 1_02]
										_snake_case :			int  						=   tokenizer(lowerCamelCase_ ,				padding=lowerCamelCase_ ,				return_tensors='pt'					)
										self.assertIsInstance(lowerCamelCase_ ,				lowerCamelCase_					)
										_snake_case :			int  						=   list(batch.input_ids.numpy()[0]					)
										self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_					)

										self.assertEqual((2, 9) ,				batch.input_ids.shape					)
										self.assertEqual((2, 9) ,				batch.attention_mask.shape					)







						def 		__UpperCAmelCase       (    self			:		Optional[int]					):





										'''simple docstring'''







										self.assertTrue(_is_whitespace(' '					)					)
										self.assertTrue(_is_whitespace('\t'					)					)
										self.assertTrue(_is_whitespace('\r'					)					)
										self.assertTrue(_is_whitespace('\n'					)					)
										self.assertTrue(_is_whitespace('\u00A0'					)					)

										self.assertFalse(_is_whitespace('A'					)					)
										self.assertFalse(_is_whitespace('-'					)					)







						def 		__UpperCAmelCase       (    self			:		Dict					):





										'''simple docstring'''







										self.assertTrue(_is_control('\u0005'					)					)

										self.assertFalse(_is_control('A'					)					)
										self.assertFalse(_is_control(' '					)					)
										self.assertFalse(_is_control('\t'					)					)
										self.assertFalse(_is_control('\r'					)					)







						def 		__UpperCAmelCase       (    self			:		List[Any]					):





										'''simple docstring'''







										self.assertTrue(_is_punctuation('-'					)					)
										self.assertTrue(_is_punctuation('$'					)					)
										self.assertTrue(_is_punctuation('`'					)					)
										self.assertTrue(_is_punctuation('.'					)					)

										self.assertFalse(_is_punctuation('A'					)					)
										self.assertFalse(_is_punctuation(' '					)					)







						@slow
						def 		__UpperCAmelCase       (    self			:		Union[str, Any]					):





										'''simple docstring'''







										_snake_case :			str  						=   self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'					)

										_snake_case :			List[str]  						=   tokenizer.encode('sequence builders' ,				add_special_tokens=lowerCamelCase_					)
										_snake_case :			List[Any]  						=   tokenizer.encode('multi-sequence build' ,				add_special_tokens=lowerCamelCase_					)

										_snake_case :			int  						=   tokenizer.build_inputs_with_special_tokens(lowerCamelCase_					)
										_snake_case :			int  						=   tokenizer.build_inputs_with_special_tokens(lowerCamelCase_ ,				lowerCamelCase_					)

										assert encoded_sentence == text + [1_02]
										assert encoded_pair == text + [1_02] + text_a + [1_02]













def       A__(		__lowerCAmelCase ):
				assert column_title.isupper()
				_snake_case :			List[Any]  						=   0
				_snake_case :			List[str]  						=   len(__lowerCAmelCase ) - 1
				_snake_case :			Dict  						=   0

				while index >= 0:
								_snake_case :			List[str]  						=   (ord(column_title[index] ) - 64) * pow(26	,			__lowerCAmelCase )
								answer += value
								power += 1
								index -= 1

				return answer


if __name__ == "__main__":
		from doctest import testmod

		testmod()






