





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class  SCREAMING_SNAKE_CASE_ (					lowerCAmelCase_   ,						unittest.TestCase			):
  '''simple docstring'''


  lowercase		:     Dict            =				GPTSanJapaneseTokenizer
  lowercase		:     str            =				False
  lowercase		:     Optional[int]            =				{"do_clean_text": False, "add_prefix_space": False}







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				Tuple:
         super().setUp()

         # fmt: off
         A						:     Dict         =['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
         # fmt: on
         A						:     Union[str, Any]         ={'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
         A						:     Optional[int]         ={'unk_token': '<unk>'}

         A						:     Optional[int]         =os.path.join(self.tmpdirname      ,	VOCAB_FILES_NAMES['vocab_file']     )
         A						:     int         =os.path.join(self.tmpdirname      ,	VOCAB_FILES_NAMES['emoji_file']     )
         with open(self.vocab_file      ,	'w'      ,	encoding='utf-8'     ) as vocab_writer:
                vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )
         with open(self.emoji_file      ,	'w'     ) as emoji_writer:
                emoji_writer.write(json.dumps(SCREAMING_SNAKE_CASE__     )     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]      ,	**SCREAMING_SNAKE_CASE__    :					Optional[Any]     )    ->				Union[str, Any]:
         kwargs.update(self.special_tokens_map     )
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname      ,	**SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]      ,	SCREAMING_SNAKE_CASE__    :					str     )    ->				List[Any]:
         A						:     Dict         ='こんにちは、世界。 \nこんばんは、㔺界。😀'
         A						:     Union[str, Any]         ='こんにちは、世界。 \nこんばんは、世界。😀'
         return input_text, output_text







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]      ,	SCREAMING_SNAKE_CASE__    :					Tuple     )    ->				Optional[Any]:
         A						:     Optional[int]         =self.get_input_output_texts(SCREAMING_SNAKE_CASE__     )
         A						:     Union[str, Any]         =tokenizer.encode(SCREAMING_SNAKE_CASE__      ,	add_special_tokens=SCREAMING_SNAKE_CASE__     )
         A						:     Any         =tokenizer.decode(SCREAMING_SNAKE_CASE__      ,	clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__     )
         return text, ids







  def 		SCREAMING_SNAKE_CASE_		(				self    :					int     )    ->				Union[str, Any]:
         pass  # TODO add if relevant







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				str:
         pass  # TODO add if relevant







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]     )    ->				Tuple:
         pass  # TODO add if relevant







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]     )    ->				Dict:
         A						:     int         =self.get_tokenizer()

         # Testing tokenization
         A						:     str         ='こんにちは、世界。　こんばんは、㔺界。'
         A						:     Any         =['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
         A						:     Any         =tokenizer.tokenize(SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

         # Testing conversion to ids without special tokens
         A						:     Any         =[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         A						:     Union[str, Any]         =tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

         # Testing conversion to ids with special tokens
         A						:     Optional[int]         =tokens + [tokenizer.unk_token]
         A						:     int         =[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
         A						:     int         =tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]     )    ->				str:
         A						:     int         =self.get_tokenizer()

         # Testing tokenization
         A						:     Any         ='こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
         A						:     Optional[Any]         ='こんにちは、、、、世界。こんばんは、、、、世界。'
         A						:     Optional[Any]         =tokenizer.encode(SCREAMING_SNAKE_CASE__     )
         A						:     int         =tokenizer.decode(SCREAMING_SNAKE_CASE__     )
         self.assertEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]     )    ->				int:
         A						:     Union[str, Any]         =self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'     )

         # Testing tokenization
         A						:     List[str]         ='こんにちは、世界。'
         A						:     List[Any]         ='こんばんは、㔺界。😀'
         A						:     Dict         ='こんにちは、世界。こんばんは、世界。😀'
         A						:     List[Any]         =tokenizer.encode(prefix_text + input_text     )
         A						:     Optional[int]         =tokenizer.encode(''      ,	prefix_text=prefix_text + input_text     )
         A						:     List[str]         =tokenizer.encode(SCREAMING_SNAKE_CASE__      ,	prefix_text=SCREAMING_SNAKE_CASE__     )
         A						:     str         =tokenizer.decode(SCREAMING_SNAKE_CASE__     )
         A						:     Tuple         =tokenizer.decode(SCREAMING_SNAKE_CASE__     )
         A						:     List[Any]         =tokenizer.decode(SCREAMING_SNAKE_CASE__     )
         self.assertEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         self.assertEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         self.assertEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]     )    ->				List[Any]:
         A						:     str         =self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'     )

         # Testing tokenization
         A						:     int         ='こんにちは、世界。'
         A						:     Optional[int]         ='こんばんは、㔺界。😀'

         A						:     str         =len(tokenizer.encode(SCREAMING_SNAKE_CASE__     )     ) - 2
         A						:     Optional[int]         =len(tokenizer.encode(SCREAMING_SNAKE_CASE__     )     ) - 2

         A						:     Union[str, Any]         =[1] + [0] * (len_prefix + len_text + 1)
         A						:     str         =[1] * (len_prefix + len_text + 1) + [0]
         A						:     str         =[1] + [1] * (len_prefix) + [0] * (len_text + 1)

         A						:     Union[str, Any]         =tokenizer(prefix_text + input_text     ).token_type_ids
         A						:     Tuple         =tokenizer(''      ,	prefix_text=prefix_text + input_text     ).token_type_ids
         A						:     str         =tokenizer(SCREAMING_SNAKE_CASE__      ,	prefix_text=SCREAMING_SNAKE_CASE__     ).token_type_ids
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]     )    ->				Optional[Any]:
         A						:     Tuple         =self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'     )

         A						:     int         =tokenizer.encode('あンいワ'     )
         A						:     int         =tokenizer.encode(''      ,	prefix_text='あンいワ'     )
         A						:     int         =tokenizer.encode('いワ'      ,	prefix_text='あン'     )

         self.assertEqual(tokenizer.decode(SCREAMING_SNAKE_CASE__     )      ,	tokenizer.decode(SCREAMING_SNAKE_CASE__     )     )
         self.assertEqual(tokenizer.decode(SCREAMING_SNAKE_CASE__     )      ,	tokenizer.decode(SCREAMING_SNAKE_CASE__     )     )
         self.assertNotEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         self.assertNotEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         self.assertEqual(x_token_a[1]      ,	x_token_a[-1]     )  # SEG token
         self.assertEqual(x_token_a[1]      ,	x_token_a[3]     )  # SEG token







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					str     )    ->				Tuple:
         A						:     str         =self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'     )

         A						:     str         =[['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
         A						:     Optional[int]         =tokenizer(SCREAMING_SNAKE_CASE__      ,	padding=SCREAMING_SNAKE_CASE__     )
         A						:     str         =tokenizer.batch_encode_plus(SCREAMING_SNAKE_CASE__      ,	padding=SCREAMING_SNAKE_CASE__     )

         # fmt: off
         A						:     Dict         =[[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
         A						:     int         =[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         A						:     int         =[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids      ,	SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(x_token.token_type_ids      ,	SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(x_token.attention_mask      ,	SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(x_token_a.input_ids      ,	SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(x_token_a.token_type_ids      ,	SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(x_token_a.attention_mask      ,	SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]     )    ->				int:
         # Intentionally convert some words to accommodate character fluctuations unique to Japanese
         pass







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]     )    ->				Dict:
         # tokenizer has no padding token
         pass







from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase							:      int      							=logging.get_logger(__name__)

_lowercase							:      Dict      							={
    '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/config.json''',
    # See all XGLM models at https://huggingface.co/models?filter=xglm
}



class  SCREAMING_SNAKE_CASE_ (					lowerCAmelCase_			):
  '''simple docstring'''


  lowercase		:     Optional[int]            =				"xglm"
  lowercase		:     Any            =				["past_key_values"]

  lowercase		:     Dict            =				{
      "num_attention_heads": "attention_heads",
      "hidden_size": "d_model",
      "num_hidden_layers": "num_layers",
  }







  def __init__(				self    :					int      ,	SCREAMING_SNAKE_CASE__    :					List[Any]=25_60_08      ,	SCREAMING_SNAKE_CASE__    :					Dict=20_48      ,	SCREAMING_SNAKE_CASE__    :					List[Any]=10_24      ,	SCREAMING_SNAKE_CASE__    :					str=40_96      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]=24      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]=16      ,	SCREAMING_SNAKE_CASE__    :					Any="gelu"      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]=0.1      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]=0.1      ,	SCREAMING_SNAKE_CASE__    :					List[Any]=0.0      ,	SCREAMING_SNAKE_CASE__    :					Tuple=0.0      ,	SCREAMING_SNAKE_CASE__    :					List[Any]=0.0_2      ,	SCREAMING_SNAKE_CASE__    :					int=True      ,	SCREAMING_SNAKE_CASE__    :					Dict=True      ,	SCREAMING_SNAKE_CASE__    :					Any=2      ,	SCREAMING_SNAKE_CASE__    :					List[Any]=1      ,	SCREAMING_SNAKE_CASE__    :					str=0      ,	SCREAMING_SNAKE_CASE__    :					List[str]=2      ,	**SCREAMING_SNAKE_CASE__    :					Dict      ,	)    ->				int:
         A						:     str         =vocab_size
         A						:     Union[str, Any]         =max_position_embeddings
         A						:     Optional[Any]         =d_model
         A						:     Optional[int]         =ffn_dim
         A						:     int         =num_layers
         A						:     Any         =attention_heads
         A						:     Dict         =activation_function
         A						:     List[Any]         =dropout
         A						:     str         =attention_dropout
         A						:     List[Any]         =activation_dropout
         A						:     List[Any]         =layerdrop
         A						:     List[Any]         =init_std
         A						:     Union[str, Any]         =scale_embedding  # scale factor will be sqrt(d_model) if True
         A						:     List[str]         =use_cache

         super().__init__(
             pad_token_id=SCREAMING_SNAKE_CASE__      ,	bos_token_id=SCREAMING_SNAKE_CASE__      ,	eos_token_id=SCREAMING_SNAKE_CASE__      ,	decoder_start_token_id=SCREAMING_SNAKE_CASE__      ,	**SCREAMING_SNAKE_CASE__      ,	)






