






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class    _UpperCAmelCase      ( A__	,unittest.TestCase			):

				"""simple docstring"""




				lowercase__       							= CodeGenTokenizer
				lowercase__       							= CodeGenTokenizerFast
				lowercase__       							= True
				lowercase__       							= {"""add_prefix_space""": True}
				lowercase__       							= False




				def        lowercase__     (  self		:					Union[str, Any]						):

								'''simple docstring'''
								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								lowercase__     				=    [
								    '''l''',
								    '''o''',
								    '''w''',
								    '''e''',
								    '''r''',
								    '''s''',
								    '''t''',
								    '''i''',
								    '''d''',
								    '''n''',
								    '''\u0120''',
								    '''\u0120l''',
								    '''\u0120n''',
								    '''\u0120lo''',
								    '''\u0120low''',
								    '''er''',
								    '''\u0120lowest''',
								    '''\u0120newer''',
								    '''\u0120wider''',
								    '''<unk>''',
								    '''<|endoftext|>''',
								]
								lowercase__     				=    dict(zip(lowerCamelCase,						range(len(lowerCamelCase						)						)						)						)
								lowercase__     				=    ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
								lowercase__     				=    {'''unk_token''': '''<unk>'''}

								lowercase__     				=    os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES['''vocab_file''']						)
								lowercase__     				=    os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES['''merges_file''']						)
								with open(self.vocab_file,						'''w''',						encoding='''utf-8'''						) as fp:
												fp.write(json.dumps(lowerCamelCase						) + '''\n'''						)
								with open(self.merges_file,						'''w''',						encoding='''utf-8'''						) as fp:
												fp.write('''\n'''.join(lowerCamelCase						)						)




				def        lowercase__     (  self		:					Tuple,						**lowerCamelCase		:					List[Any]						):

								'''simple docstring'''
								kwargs.update(self.special_tokens_map						)
								return CodeGenTokenizer.from_pretrained(self.tmpdirname,						**lowerCamelCase						)




				def        lowercase__     (  self		:					List[Any],						**lowerCamelCase		:					List[str]						):

								'''simple docstring'''
								kwargs.update(self.special_tokens_map						)
								return CodeGenTokenizerFast.from_pretrained(self.tmpdirname,						**lowerCamelCase						)




				def        lowercase__     (  self		:					Dict,						lowerCamelCase		:					Dict						):

								'''simple docstring'''
								lowercase__     				=    '''lower newer'''
								lowercase__     				=    '''lower newer'''
								return input_text, output_text




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								lowercase__     				=    CodeGenTokenizer(self.vocab_file,						self.merges_file,						**self.special_tokens_map						)
								lowercase__     				=    '''lower newer'''
								lowercase__     				=    ['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
								lowercase__     				=    tokenizer.tokenize(lowerCamelCase,						add_prefix_space=lowerCamelCase						)
								self.assertListEqual(lowerCamelCase,						lowerCamelCase						)

								lowercase__     				=    tokens + [tokenizer.unk_token]
								lowercase__     				=    [14, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						),						lowerCamelCase						)




				def        lowercase__     (  self		:					List[Any]						):

								'''simple docstring'''
								if not self.test_rust_tokenizer:
												return

								lowercase__     				=    self.get_tokenizer()
								lowercase__     				=    self.get_rust_tokenizer(add_prefix_space=lowerCamelCase						)

								lowercase__     				=    '''lower newer'''

								# Testing tokenization
								lowercase__     				=    tokenizer.tokenize(lowerCamelCase,						add_prefix_space=lowerCamelCase						)
								lowercase__     				=    rust_tokenizer.tokenize(lowerCamelCase						)
								self.assertListEqual(lowerCamelCase,						lowerCamelCase						)

								# Testing conversion to ids without special tokens
								lowercase__     				=    tokenizer.encode(lowerCamelCase,						add_special_tokens=lowerCamelCase,						add_prefix_space=lowerCamelCase						)
								lowercase__     				=    rust_tokenizer.encode(lowerCamelCase,						add_special_tokens=lowerCamelCase						)
								self.assertListEqual(lowerCamelCase,						lowerCamelCase						)

								# Testing conversion to ids with special tokens
								lowercase__     				=    self.get_rust_tokenizer(add_prefix_space=lowerCamelCase						)
								lowercase__     				=    tokenizer.encode(lowerCamelCase,						add_prefix_space=lowerCamelCase						)
								lowercase__     				=    rust_tokenizer.encode(lowerCamelCase						)
								self.assertListEqual(lowerCamelCase,						lowerCamelCase						)

								# Testing the unknown token
								lowercase__     				=    tokens + [rust_tokenizer.unk_token]
								lowercase__     				=    [14, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCamelCase						),						lowerCamelCase						)




				def        lowercase__     (  self		:					List[str],						*lowerCamelCase		:					str,						**lowerCamelCase		:					Dict						):

								'''simple docstring'''
								# It's very difficult to mix/test pretokenization with byte-level
								# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
								pass




				def        lowercase__     (  self		:					Any,						lowerCamelCase		:					Any=15						):

								'''simple docstring'''
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
																lowercase__     				=    self.rust_tokenizer_class.from_pretrained(lowerCamelCase,						**lowerCamelCase						)

																# Simple input
																lowercase__     				=    '''This is a simple input'''
																lowercase__     				=    ['''This is a simple input 1''', '''This is a simple input 2''']
																lowercase__     				=    ('''This is a simple input''', '''This is a pair''')
																lowercase__     				=    [
																    ('''This is a simple input 1''', '''This is a simple input 2'''),
																    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																]

																# Simple input tests
																self.assertRaises(lowerCamelCase,						tokenizer_r.encode,						lowerCamelCase,						max_length=lowerCamelCase,						padding='''max_length'''						)

																# Simple input
																self.assertRaises(lowerCamelCase,						tokenizer_r.encode_plus,						lowerCamelCase,						max_length=lowerCamelCase,						padding='''max_length'''						)

																# Simple input
																self.assertRaises(
																    lowerCamelCase,						tokenizer_r.batch_encode_plus,						lowerCamelCase,						max_length=lowerCamelCase,						padding='''max_length''',						)

																# Pair input
																self.assertRaises(lowerCamelCase,						tokenizer_r.encode,						lowerCamelCase,						max_length=lowerCamelCase,						padding='''max_length'''						)

																# Pair input
																self.assertRaises(lowerCamelCase,						tokenizer_r.encode_plus,						lowerCamelCase,						max_length=lowerCamelCase,						padding='''max_length'''						)

																# Pair input
																self.assertRaises(
																    lowerCamelCase,						tokenizer_r.batch_encode_plus,						lowerCamelCase,						max_length=lowerCamelCase,						padding='''max_length''',						)




				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								lowercase__     				=    CodeGenTokenizer.from_pretrained(self.tmpdirname,						pad_token='''<pad>'''						)

								# Simple input
								lowercase__     				=    '''This is a simple input'''
								lowercase__     				=    ['''This is a simple input looooooooong''', '''This is a simple input''']
								lowercase__     				=    ('''This is a simple input''', '''This is a pair''')
								lowercase__     				=    [
								    ('''This is a simple input loooooong''', '''This is a simple input'''),
								    ('''This is a simple pair loooooong''', '''This is a simple pair'''),
								]

								lowercase__     				=    tokenizer.pad_token_id

								lowercase__     				=    tokenizer(lowerCamelCase,						padding='''max_length''',						max_length=30,						return_tensors='''np'''						)
								lowercase__     				=    tokenizer(lowerCamelCase,						padding=lowerCamelCase,						truncate=lowerCamelCase,						return_tensors='''np'''						)
								lowercase__     				=    tokenizer(*lowerCamelCase,						padding='''max_length''',						max_length=60,						return_tensors='''np'''						)
								lowercase__     				=    tokenizer(lowerCamelCase,						padding=lowerCamelCase,						truncate=lowerCamelCase,						return_tensors='''np'''						)

								# s
								# test single string max_length padding
								self.assertEqual(out_s['''input_ids'''].shape[-1],						30						)
								self.assertTrue(pad_token_id in out_s['''input_ids''']						)
								self.assertTrue(0 in out_s['''attention_mask''']						)

								# s2
								# test automatic padding
								self.assertEqual(out_sa['''input_ids'''].shape[-1],						33						)
								# long slice doesn't have padding
								self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]						)
								self.assertFalse(0 in out_sa['''attention_mask'''][0]						)
								# short slice does have padding
								self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]						)
								self.assertTrue(0 in out_sa['''attention_mask'''][1]						)

								# p
								# test single pair max_length padding
								self.assertEqual(out_p['''input_ids'''].shape[-1],						60						)
								self.assertTrue(pad_token_id in out_p['''input_ids''']						)
								self.assertTrue(0 in out_p['''attention_mask''']						)

								# p2
								# test automatic padding pair
								self.assertEqual(out_pa['''input_ids'''].shape[-1],						52						)
								# long slice pair doesn't have padding
								self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]						)
								self.assertFalse(0 in out_pa['''attention_mask'''][0]						)
								# short slice pair does have padding
								self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]						)
								self.assertTrue(0 in out_pa['''attention_mask'''][1]						)




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								lowercase__     				=    '''$$$'''
								lowercase__     				=    CodeGenTokenizer.from_pretrained(self.tmpdirname,						bos_token=lowerCamelCase,						add_bos_token=lowerCamelCase						)

								lowercase__     				=    '''This is a simple input'''
								lowercase__     				=    ['''This is a simple input 1''', '''This is a simple input 2''']

								lowercase__     				=    tokenizer.bos_token_id

								lowercase__     				=    tokenizer(lowerCamelCase						)
								lowercase__     				=    tokenizer(lowerCamelCase						)

								self.assertEqual(out_s.input_ids[0],						lowerCamelCase						)
								self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

								lowercase__     				=    tokenizer.decode(out_s.input_ids						)
								lowercase__     				=    tokenizer.batch_decode(out_sa.input_ids						)

								self.assertEqual(decode_s.split()[0],						lowerCamelCase						)
								self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)




				@slow
				def        lowercase__     (  self		:					int						):

								'''simple docstring'''
								lowercase__     				=    CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''						)

								lowercase__     				=    '''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
								lowercase__     				=    '''\nif len_a > len_b:      result = a\nelse:      result = b'''

								lowercase__     				=    tokenizer.encode(lowerCamelCase						)
								lowercase__     				=    ['''^#''', re.escape('''<|endoftext|>'''						), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
								lowercase__     				=    tokenizer.decode(lowerCamelCase,						truncate_before_pattern=lowerCamelCase						)
								self.assertEqual(lowerCamelCase,						lowerCamelCase						)




				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								pass












from transformers import DistilBertTokenizer, DistilBertTokenizerFast
from transformers.testing_utils import require_tokenizers, slow

from ..bert.test_tokenization_bert import BertTokenizationTest
@require_tokenizers
class    _UpperCAmelCase      ( A__			):

				"""simple docstring"""




				lowercase__       							= DistilBertTokenizer
				lowercase__       							= DistilBertTokenizerFast
				lowercase__       							= True




				@slow
				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								lowercase__     				=    DistilBertTokenizer.from_pretrained('''distilbert-base-uncased'''						)

								lowercase__     				=    tokenizer.encode('''sequence builders''',						add_special_tokens=lowerCamelCase						)
								lowercase__     				=    tokenizer.encode('''multi-sequence build''',						add_special_tokens=lowerCamelCase						)

								lowercase__     				=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase						)
								lowercase__     				=    tokenizer.build_inputs_with_special_tokens(lowerCamelCase,						lowerCamelCase						)

								assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
								assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
								    tokenizer.sep_token_id
								]




