



'''simple docstring'''
import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class      __SCREAMING_SNAKE_CASE      (			__a	,				unittest.TestCase				):
 snake_case_  							=							GPTaTokenizer
 snake_case_  							=							GPTaTokenizerFast
 snake_case_  							=							True
 snake_case_  							=							{"add_prefix_space": True}
 snake_case_  							=							False




 def      __magic_name__	(      self	:							Optional[int]      )	->						Dict:
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       SCREAMING_SNAKE_CASE__     :				Any									=[
           '''l''',
           '''o''',
           '''w''',
           '''e''',
           '''r''',
           '''s''',
           '''t''',
           '''i''',
           '''d''',
           '''n''',
           '''\u0120''',
           '''\u0120l''',
           '''\u0120n''',
           '''\u0120lo''',
           '''\u0120low''',
           '''er''',
           '''\u0120lowest''',
           '''\u0120newer''',
           '''\u0120wider''',
           '''<unk>''',
           '''<|endoftext|>''',
       ]
       SCREAMING_SNAKE_CASE__     :				List[Any]									=dict(zip(lowerCAmelCase_					,      range(len(lowerCAmelCase_      )      )      )      )
       SCREAMING_SNAKE_CASE__     :				str									=['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
       SCREAMING_SNAKE_CASE__     :				Any									={'''unk_token''': '''<unk>'''}

       SCREAMING_SNAKE_CASE__     :				str									=os.path.join(self.tmpdirname					,      VOCAB_FILES_NAMES['''vocab_file''']      )
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=os.path.join(self.tmpdirname					,      VOCAB_FILES_NAMES['''merges_file''']      )
       with open(self.vocab_file					,      '''w'''					,      encoding='''utf-8'''      ) as fp:
             fp.write(json.dumps(lowerCAmelCase_      ) + '''\n'''      )
       with open(self.merges_file					,      '''w'''					,      encoding='''utf-8'''      ) as fp:
             fp.write('''\n'''.join(lowerCAmelCase_      )      )




 def      __magic_name__	(      self	:							Optional[int]					,      **__lowercase	:							int      )	->						str:
       kwargs.update(self.special_tokens_map      )
       return GPTaTokenizer.from_pretrained(self.tmpdirname					,      **lowerCAmelCase_      )




 def      __magic_name__	(      self	:							Union[str, Any]					,      **__lowercase	:							Union[str, Any]      )	->						List[str]:
       kwargs.update(self.special_tokens_map      )
       return GPTaTokenizerFast.from_pretrained(self.tmpdirname					,      **lowerCAmelCase_      )




 def      __magic_name__	(      self	:							List[Any]					,      __lowercase	:							int      )	->						List[Any]:
       SCREAMING_SNAKE_CASE__     :				str									='''lower newer'''
       SCREAMING_SNAKE_CASE__     :				str									='''lower newer'''
       return input_text, output_text




 def      __magic_name__	(      self	:							Union[str, Any]      )	->						Optional[int]:
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=GPTaTokenizer(self.vocab_file					,      self.merges_file					,      **self.special_tokens_map      )
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									='''lower newer'''
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=tokenizer.tokenize(lowerCAmelCase_					,      add_prefix_space=lowerCAmelCase_      )
       self.assertListEqual(lowerCAmelCase_					,      lowerCAmelCase_      )

       SCREAMING_SNAKE_CASE__     :				List[Any]									=tokens + [tokenizer.unk_token]
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=[14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_      )					,      lowerCAmelCase_      )




 def      __magic_name__	(      self	:							Optional[Any]      )	->						Any:







       if not self.test_rust_tokenizer:
             return

       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=self.get_tokenizer()
       SCREAMING_SNAKE_CASE__     :				Tuple									=self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_      )

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									='''lower newer'''

       # Testing tokenization
       SCREAMING_SNAKE_CASE__     :				str									=tokenizer.tokenize(lowerCAmelCase_					,      add_prefix_space=lowerCAmelCase_      )
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=rust_tokenizer.tokenize(lowerCAmelCase_      )
       self.assertListEqual(lowerCAmelCase_					,      lowerCAmelCase_      )

       # Testing conversion to ids without special tokens
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=tokenizer.encode(lowerCAmelCase_					,      add_special_tokens=lowerCAmelCase_					,      add_prefix_space=lowerCAmelCase_      )
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=rust_tokenizer.encode(lowerCAmelCase_					,      add_special_tokens=lowerCAmelCase_      )
       self.assertListEqual(lowerCAmelCase_					,      lowerCAmelCase_      )

       # Testing conversion to ids with special tokens
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_      )
       SCREAMING_SNAKE_CASE__     :				str									=tokenizer.encode(lowerCAmelCase_					,      add_prefix_space=lowerCAmelCase_      )
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=rust_tokenizer.encode(lowerCAmelCase_      )
       self.assertListEqual(lowerCAmelCase_					,      lowerCAmelCase_      )

       # Testing the unknown token
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=tokens + [rust_tokenizer.unk_token]
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=[14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCAmelCase_      )					,      lowerCAmelCase_      )




 def      __magic_name__	(      self	:							Dict					,      *__lowercase	:							Optional[Any]					,      **__lowercase	:							List[str]      )	->						Optional[int]:
       pass




 def      __magic_name__	(      self	:							List[str]					,      __lowercase	:							List[str]=15      )	->						int:







       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
                   SCREAMING_SNAKE_CASE__     :				List[Any]									=self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_					,      **lowerCAmelCase_      )

                   # Simple input
                   SCREAMING_SNAKE_CASE__     :				Any									='''This is a simple input'''
                   SCREAMING_SNAKE_CASE__     :				List[Any]									=['''This is a simple input 1''', '''This is a simple input 2''']
                   SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=('''This is a simple input''', '''This is a pair''')
                   SCREAMING_SNAKE_CASE__     :				List[Any]									=[
                       ('''This is a simple input 1''', '''This is a simple input 2'''),
                       ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                   ]

                   # Simple input tests
                   self.assertRaises(lowerCAmelCase_					,      tokenizer_r.encode					,      lowerCAmelCase_					,      max_length=lowerCAmelCase_					,      padding='''max_length'''      )

                   # Simple input
                   self.assertRaises(lowerCAmelCase_					,      tokenizer_r.encode_plus					,      lowerCAmelCase_					,      max_length=lowerCAmelCase_					,      padding='''max_length'''      )

                   # Simple input
                   self.assertRaises(
                       lowerCAmelCase_					,      tokenizer_r.batch_encode_plus					,      lowerCAmelCase_					,      max_length=lowerCAmelCase_					,      padding='''max_length'''					,      )

                   # Pair input
                   self.assertRaises(lowerCAmelCase_					,      tokenizer_r.encode					,      lowerCAmelCase_					,      max_length=lowerCAmelCase_					,      padding='''max_length'''      )

                   # Pair input
                   self.assertRaises(lowerCAmelCase_					,      tokenizer_r.encode_plus					,      lowerCAmelCase_					,      max_length=lowerCAmelCase_					,      padding='''max_length'''      )

                   # Pair input
                   self.assertRaises(
                       lowerCAmelCase_					,      tokenizer_r.batch_encode_plus					,      lowerCAmelCase_					,      max_length=lowerCAmelCase_					,      padding='''max_length'''					,      )




 def      __magic_name__	(      self	:							Union[str, Any]      )	->						Optional[Any]:
       SCREAMING_SNAKE_CASE__     :				int									=GPTaTokenizer.from_pretrained(self.tmpdirname					,      pad_token='''<pad>'''      )

       # Simple input
       SCREAMING_SNAKE_CASE__     :				int									='''This is a simple input'''
       SCREAMING_SNAKE_CASE__     :				int									=['''This is a simple input looooooooong''', '''This is a simple input''']
       SCREAMING_SNAKE_CASE__     :				Tuple									=('''This is a simple input''', '''This is a pair''')
       SCREAMING_SNAKE_CASE__     :				int									=[
           ('''This is a simple input loooooong''', '''This is a simple input'''),
           ('''This is a simple pair loooooong''', '''This is a simple pair'''),
       ]

       SCREAMING_SNAKE_CASE__     :				List[str]									=tokenizer.pad_token_id

       SCREAMING_SNAKE_CASE__     :				Optional[int]									=tokenizer(lowerCAmelCase_					,      padding='''max_length'''					,      max_length=30					,      return_tensors='''np'''      )
       SCREAMING_SNAKE_CASE__     :				str									=tokenizer(lowerCAmelCase_					,      padding=lowerCAmelCase_					,      truncate=lowerCAmelCase_					,      return_tensors='''np'''      )
       SCREAMING_SNAKE_CASE__     :				int									=tokenizer(*lowerCAmelCase_					,      padding='''max_length'''					,      max_length=60					,      return_tensors='''np'''      )
       SCREAMING_SNAKE_CASE__     :				List[Any]									=tokenizer(lowerCAmelCase_					,      padding=lowerCAmelCase_					,      truncate=lowerCAmelCase_					,      return_tensors='''np'''      )

       # s
       # test single string max_length padding
       self.assertEqual(out_s['''input_ids'''].shape[-1]					,      30      )
       self.assertTrue(pad_token_id in out_s['''input_ids''']      )
       self.assertTrue(0 in out_s['''attention_mask''']      )

       # s2
       # test automatic padding
       self.assertEqual(out_sa['''input_ids'''].shape[-1]					,      33      )
       # long slice doesn't have padding
       self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]      )
       self.assertFalse(0 in out_sa['''attention_mask'''][0]      )
       # short slice does have padding
       self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]      )
       self.assertTrue(0 in out_sa['''attention_mask'''][1]      )

       # p
       # test single pair max_length padding
       self.assertEqual(out_p['''input_ids'''].shape[-1]					,      60      )
       self.assertTrue(pad_token_id in out_p['''input_ids''']      )
       self.assertTrue(0 in out_p['''attention_mask''']      )

       # p2
       # test automatic padding pair
       self.assertEqual(out_pa['''input_ids'''].shape[-1]					,      52      )
       # long slice pair doesn't have padding
       self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]      )
       self.assertFalse(0 in out_pa['''attention_mask'''][0]      )
       # short slice pair does have padding
       self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]      )
       self.assertTrue(0 in out_pa['''attention_mask'''][1]      )




 def      __magic_name__	(      self	:							List[Any]      )	->						List[Any]:
       SCREAMING_SNAKE_CASE__     :				str									='''$$$'''
       SCREAMING_SNAKE_CASE__     :				Any									=GPTaTokenizer.from_pretrained(self.tmpdirname					,      bos_token=lowerCAmelCase_					,      add_bos_token=lowerCAmelCase_      )

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									='''This is a simple input'''
       SCREAMING_SNAKE_CASE__     :				int									=['''This is a simple input 1''', '''This is a simple input 2''']

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=tokenizer.bos_token_id

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=tokenizer(lowerCAmelCase_      )
       SCREAMING_SNAKE_CASE__     :				Tuple									=tokenizer(lowerCAmelCase_      )

       self.assertEqual(out_s.input_ids[0]					,      lowerCAmelCase_      )
       self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids      )      )

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=tokenizer.decode(out_s.input_ids      )
       SCREAMING_SNAKE_CASE__     :				Dict									=tokenizer.batch_decode(out_sa.input_ids      )

       self.assertEqual(decode_s.split()[0]					,      lowerCAmelCase_      )
       self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa      )      )




 def      __magic_name__	(      self	:							List[str]      )	->						Optional[int]:
       pass




 def      __magic_name__	(      self	:							Any      )	->						Any:
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=[self.get_tokenizer(do_lower_case=lowerCAmelCase_					,      add_bos_token=lowerCAmelCase_      )]
       for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"      ):
                   SCREAMING_SNAKE_CASE__     :				str									='''Encode this.'''
                   SCREAMING_SNAKE_CASE__     :				int									='''This one too please.'''
                   SCREAMING_SNAKE_CASE__     :				Optional[int]									=tokenizer.encode(lowerCAmelCase_					,      add_special_tokens=lowerCAmelCase_      )
                   encoded_sequence += tokenizer.encode(lowerCAmelCase_					,      add_special_tokens=lowerCAmelCase_      )
                   SCREAMING_SNAKE_CASE__     :				Dict									=tokenizer.encode_plus(
                       lowerCAmelCase_					,      lowerCAmelCase_					,      add_special_tokens=lowerCAmelCase_					,      return_special_tokens_mask=lowerCAmelCase_					,      )
                   SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=encoded_sequence_dict['''input_ids''']
                   SCREAMING_SNAKE_CASE__     :				Tuple									=encoded_sequence_dict['''special_tokens_mask''']
                   self.assertEqual(len(lowerCAmelCase_      )					,      len(lowerCAmelCase_      )      )

                   SCREAMING_SNAKE_CASE__     :				Any									=[
                       (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowerCAmelCase_      )
                   ]
                   SCREAMING_SNAKE_CASE__     :				str									=[x for x in filtered_sequence if x is not None]
                   self.assertEqual(lowerCAmelCase_					,      lowerCAmelCase_      )






@require_tokenizers
class      __SCREAMING_SNAKE_CASE      (			unittest.TestCase				):




 def      __magic_name__	(      self	:							List[str]      )	->						List[str]:
       SCREAMING_SNAKE_CASE__     :				Dict									=AutoTokenizer.from_pretrained('''facebook/opt-350m'''					,      from_slow=lowerCAmelCase_      )
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									='''A photo of a cat'''

       SCREAMING_SNAKE_CASE__     :				Dict									=tokenizer.encode(
           lowerCAmelCase_					,      )
       self.assertEqual(lowerCAmelCase_					,      [2, 2_50, 13_45, 9, 10, 47_58]      )
       tokenizer.save_pretrained('''test_opt'''      )

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=AutoTokenizer.from_pretrained('''./test_opt'''      )
       SCREAMING_SNAKE_CASE__     :				Any									=tokenizer.encode(
           lowerCAmelCase_					,      )
       self.assertEqual(lowerCAmelCase_					,      [2, 2_50, 13_45, 9, 10, 47_58]      )




 def      __magic_name__	(      self	:							int      )	->						Optional[int]:
       SCREAMING_SNAKE_CASE__     :				List[Any]									=AutoTokenizer.from_pretrained('''facebook/opt-350m'''					,      use_slow=lowerCAmelCase_      )
       SCREAMING_SNAKE_CASE__     :				str									='''A photo of a cat'''

       SCREAMING_SNAKE_CASE__     :				List[str]									=tokenizer.encode(
           lowerCAmelCase_					,      )
       # Same as above
       self.assertEqual(lowerCAmelCase_					,      [2, 2_50, 13_45, 9, 10, 47_58]      )




 @unittest.skip('''This test is failing because of a bug in the fast tokenizer'''      )
 def      __magic_name__	(      self	:							Dict      )	->						str:
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=AutoTokenizer.from_pretrained('''facebook/opt-350m'''					,      from_slow=lowerCAmelCase_      )

       SCREAMING_SNAKE_CASE__     :				List[Any]									='''bos'''
       SCREAMING_SNAKE_CASE__     :				List[Any]									=tokenizer.get_vocab()['''bos''']

       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									='''A photo of a cat'''
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=tokenizer.encode(
           lowerCAmelCase_					,      )
       # We changed the bos token
       self.assertEqual(lowerCAmelCase_					,      [3_19_57, 2_50, 13_45, 9, 10, 47_58]      )
       tokenizer.save_pretrained('''./tok'''      )
       SCREAMING_SNAKE_CASE__     :				List[Any]									=AutoTokenizer.from_pretrained('''./tok'''      )
       self.assertTrue(tokenizer.is_fast      )
       SCREAMING_SNAKE_CASE__     :				Dict									=tokenizer.encode(
           lowerCAmelCase_					,      )
       self.assertEqual(lowerCAmelCase_					,      [3_19_57, 2_50, 13_45, 9, 10, 47_58]      )







'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_					= logging.get_logger(__name__)

a_					= {
    'bigcode/gpt_bigcode-santacoder': 'https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json',
}






class      __SCREAMING_SNAKE_CASE      (			lowerCamelCase				):
 snake_case_  							=							"""gpt_bigcode"""
 snake_case_  							=							["""past_key_values"""]
 snake_case_  							=							{
     """hidden_size""": """n_embd""",
     """max_position_embeddings""": """n_positions""",
     """num_attention_heads""": """n_head""",
     """num_hidden_layers""": """n_layer""",
 }




 def __init__(      self	:							Any					,      __lowercase	:							Any=5_02_57					,      __lowercase	:							int=10_24					,      __lowercase	:							List[str]=7_68					,      __lowercase	:							Optional[int]=12					,      __lowercase	:							Dict=12					,      __lowercase	:							List[str]=None					,      __lowercase	:							int="gelu_pytorch_tanh"					,      __lowercase	:							Union[str, Any]=0.1					,      __lowercase	:							Optional[int]=0.1					,      __lowercase	:							Optional[int]=0.1					,      __lowercase	:							Optional[Any]=1e-5					,      __lowercase	:							List[str]=0.02					,      __lowercase	:							Tuple=True					,      __lowercase	:							Optional[Any]=True					,      __lowercase	:							Union[str, Any]=5_02_56					,      __lowercase	:							List[Any]=5_02_56					,      __lowercase	:							Union[str, Any]=True					,      __lowercase	:							List[str]=True					,      __lowercase	:							Dict=True					,      **__lowercase	:							List[Any]					,      )	->						Optional[int]:
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=vocab_size
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=n_positions
       SCREAMING_SNAKE_CASE__     :				Dict									=n_embd
       SCREAMING_SNAKE_CASE__     :				Dict									=n_layer
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=n_head
       SCREAMING_SNAKE_CASE__     :				List[str]									=n_inner
       SCREAMING_SNAKE_CASE__     :				List[str]									=activation_function
       SCREAMING_SNAKE_CASE__     :				List[Any]									=resid_pdrop
       SCREAMING_SNAKE_CASE__     :				List[Any]									=embd_pdrop
       SCREAMING_SNAKE_CASE__     :				List[str]									=attn_pdrop
       SCREAMING_SNAKE_CASE__     :				Dict									=layer_norm_epsilon
       SCREAMING_SNAKE_CASE__     :				List[str]									=initializer_range
       SCREAMING_SNAKE_CASE__     :				List[Any]									=scale_attn_weights
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=use_cache
       SCREAMING_SNAKE_CASE__     :				Dict									=attention_softmax_in_fpaa
       SCREAMING_SNAKE_CASE__     :				int									=scale_attention_softmax_in_fpaa
       SCREAMING_SNAKE_CASE__     :				Dict									=multi_query

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=bos_token_id
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=eos_token_id

       super().__init__(bos_token_id=__lowercase					,      eos_token_id=__lowercase					,      **__lowercase      )