

"""simple docstring"""




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class    lowerCamelCase						(A__			,unittest.TestCase			):
			lowerCamelCase__   :		Optional[int]							=							GPTSanJapaneseTokenizer
			lowerCamelCase__   :		List[str]							=							False
			lowerCamelCase__   :		Union[str, Any]							=							{'do_clean_text': False, 'add_prefix_space': False}

			def 					SCREAMING_SNAKE_CASE				(					self				:						int       )   ->    int:
						super().setUp()

						# fmt: off
						SCREAMING_SNAKE_CASE__								=      ["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
						# fmt: on
						SCREAMING_SNAKE_CASE__								=      {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
						SCREAMING_SNAKE_CASE__								=      {"""unk_token""": """<unk>"""}

						SCREAMING_SNAKE_CASE__								=      os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES["""vocab_file"""]       )
						SCREAMING_SNAKE_CASE__								=      os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES["""emoji_file"""]       )
						with open(self.vocab_file    ,				"""w"""    ,				encoding="""utf-8"""       ) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )
						with open(self.emoji_file    ,				"""w"""       ) as emoji_writer:
									emoji_writer.write(json.dumps(__UpperCAmelCase       )       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Any    ,				**__UpperCAmelCase				:						Union[str, Any]       )   ->    Optional[int]:
						kwargs.update(self.special_tokens_map       )
						return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname    ,				**__UpperCAmelCase       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						int    ,				__UpperCAmelCase				:						int       )   ->    List[Any]:
						SCREAMING_SNAKE_CASE__								=      """こんにちは、世界。 \nこんばんは、㔺界。😀"""
						SCREAMING_SNAKE_CASE__								=      """こんにちは、世界。 \nこんばんは、世界。😀"""
						return input_text, output_text

			def 					SCREAMING_SNAKE_CASE				(					self				:						str    ,				__UpperCAmelCase				:						Any       )   ->    List[str]:
						SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__								=      self.get_input_output_texts(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.encode(__UpperCAmelCase    ,				add_special_tokens=__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.decode(__UpperCAmelCase    ,				clean_up_tokenization_spaces=__UpperCAmelCase       )
						return text, ids

			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple       )   ->    Union[str, Any]:
						pass  # TODO add if relevant

			def 					SCREAMING_SNAKE_CASE				(					self				:						List[str]       )   ->    Optional[Any]:
						pass  # TODO add if relevant

			def 					SCREAMING_SNAKE_CASE				(					self				:						Union[str, Any]       )   ->    List[str]:
						pass  # TODO add if relevant

			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[int]       )   ->    Optional[Any]:
						SCREAMING_SNAKE_CASE__								=      self.get_tokenizer()

						# Testing tokenization
						SCREAMING_SNAKE_CASE__								=      """こんにちは、世界。　こんばんは、㔺界。"""
						SCREAMING_SNAKE_CASE__								=      ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
						SCREAMING_SNAKE_CASE__								=      tokenizer.tokenize(__UpperCAmelCase       )
						self.assertListEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )

						# Testing conversion to ids without special tokens
						SCREAMING_SNAKE_CASE__								=      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
						SCREAMING_SNAKE_CASE__								=      tokenizer.convert_tokens_to_ids(__UpperCAmelCase       )
						self.assertListEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )

						# Testing conversion to ids with special tokens
						SCREAMING_SNAKE_CASE__								=      tokens + [tokenizer.unk_token]
						SCREAMING_SNAKE_CASE__								=      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
						SCREAMING_SNAKE_CASE__								=      tokenizer.convert_tokens_to_ids(__UpperCAmelCase       )
						self.assertListEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple       )   ->    Union[str, Any]:
						SCREAMING_SNAKE_CASE__								=      self.get_tokenizer()

						# Testing tokenization
						SCREAMING_SNAKE_CASE__								=      """こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
						SCREAMING_SNAKE_CASE__								=      """こんにちは、、、、世界。こんばんは、、、、世界。"""
						SCREAMING_SNAKE_CASE__								=      tokenizer.encode(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.decode(__UpperCAmelCase       )
						self.assertEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )

			@slow
			def 					SCREAMING_SNAKE_CASE				(					self				:						str       )   ->    List[Any]:
						SCREAMING_SNAKE_CASE__								=      self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""       )

						# Testing tokenization
						SCREAMING_SNAKE_CASE__								=      """こんにちは、世界。"""
						SCREAMING_SNAKE_CASE__								=      """こんばんは、㔺界。😀"""
						SCREAMING_SNAKE_CASE__								=      """こんにちは、世界。こんばんは、世界。😀"""
						SCREAMING_SNAKE_CASE__								=      tokenizer.encode(prefix_text + input_text       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.encode(""""""    ,				prefix_text=prefix_text + input_text       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.encode(__UpperCAmelCase    ,				prefix_text=__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.decode(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.decode(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.decode(__UpperCAmelCase       )
						self.assertEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )
						self.assertEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )
						self.assertEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )

			@slow
			def 					SCREAMING_SNAKE_CASE				(					self				:						Dict       )   ->    List[Any]:
						SCREAMING_SNAKE_CASE__								=      self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""       )

						# Testing tokenization
						SCREAMING_SNAKE_CASE__								=      """こんにちは、世界。"""
						SCREAMING_SNAKE_CASE__								=      """こんばんは、㔺界。😀"""

						SCREAMING_SNAKE_CASE__								=      len(tokenizer.encode(__UpperCAmelCase       )       ) - 2
						SCREAMING_SNAKE_CASE__								=      len(tokenizer.encode(__UpperCAmelCase       )       ) - 2

						SCREAMING_SNAKE_CASE__								=      [1] + [0] * (len_prefix + len_text + 1)
						SCREAMING_SNAKE_CASE__								=      [1] * (len_prefix + len_text + 1) + [0]
						SCREAMING_SNAKE_CASE__								=      [1] + [1] * (len_prefix) + [0] * (len_text + 1)

						SCREAMING_SNAKE_CASE__								=      tokenizer(prefix_text + input_text       ).token_type_ids
						SCREAMING_SNAKE_CASE__								=      tokenizer(""""""    ,				prefix_text=prefix_text + input_text       ).token_type_ids
						SCREAMING_SNAKE_CASE__								=      tokenizer(__UpperCAmelCase    ,				prefix_text=__UpperCAmelCase       ).token_type_ids
						self.assertListEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )
						self.assertListEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )
						self.assertListEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )

			@slow
			def 					SCREAMING_SNAKE_CASE				(					self				:						str       )   ->    int:
						SCREAMING_SNAKE_CASE__								=      self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""       )

						SCREAMING_SNAKE_CASE__								=      tokenizer.encode("""あンいワ"""       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.encode(""""""    ,				prefix_text="""あンいワ"""       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.encode("""いワ"""    ,				prefix_text="""あン"""       )

						self.assertEqual(tokenizer.decode(__UpperCAmelCase       )    ,				tokenizer.decode(__UpperCAmelCase       )       )
						self.assertEqual(tokenizer.decode(__UpperCAmelCase       )    ,				tokenizer.decode(__UpperCAmelCase       )       )
						self.assertNotEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )
						self.assertNotEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )
						self.assertEqual(x_token_a[1]    ,				x_token_a[-1]       )  # SEG token
						self.assertEqual(x_token_a[1]    ,				x_token_a[3]       )  # SEG token

			@slow
			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple       )   ->    int:
						SCREAMING_SNAKE_CASE__								=      self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""       )

						SCREAMING_SNAKE_CASE__								=      [["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
						SCREAMING_SNAKE_CASE__								=      tokenizer(__UpperCAmelCase    ,				padding=__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      tokenizer.batch_encode_plus(__UpperCAmelCase    ,				padding=__UpperCAmelCase       )

						# fmt: off
						SCREAMING_SNAKE_CASE__								=      [[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
						SCREAMING_SNAKE_CASE__								=      [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
						SCREAMING_SNAKE_CASE__								=      [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
						# fmt: on
						self.assertListEqual(x_token.input_ids    ,				__UpperCAmelCase       )
						self.assertListEqual(x_token.token_type_ids    ,				__UpperCAmelCase       )
						self.assertListEqual(x_token.attention_mask    ,				__UpperCAmelCase       )
						self.assertListEqual(x_token_a.input_ids    ,				__UpperCAmelCase       )
						self.assertListEqual(x_token_a.token_type_ids    ,				__UpperCAmelCase       )
						self.assertListEqual(x_token_a.attention_mask    ,				__UpperCAmelCase       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						List[Any]       )   ->    Dict:
						# Intentionally convert some words to accommodate character fluctuations unique to Japanese
						pass





			def 					SCREAMING_SNAKE_CASE				(					self				:						int       )   ->    str:
						# tokenizer has no padding token
						pass





"""simple docstring"""




import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_							:			List[Any]											= logging.get_logger(__name__)

A_							:			Dict											= {
    "microsoft/unispeech-large-1500h-cv": (
        "https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json"
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}





class    lowerCamelCase						(A__			):
			lowerCamelCase__   :		int							=							'unispeech'

			def __init__(					self				:						Union[str, Any]    ,				__UpperCAmelCase				:						List[Any]=3_2    ,				__UpperCAmelCase				:						Union[str, Any]=7_6_8    ,				__UpperCAmelCase				:						Tuple=1_2    ,				__UpperCAmelCase				:						Dict=1_2    ,				__UpperCAmelCase				:						Optional[Any]=3_0_7_2    ,				__UpperCAmelCase				:						Optional[Any]="gelu"    ,				__UpperCAmelCase				:						Union[str, Any]=0.1    ,				__UpperCAmelCase				:						List[Any]=0.1    ,				__UpperCAmelCase				:						List[str]=0.1    ,				__UpperCAmelCase				:						List[str]=0.0    ,				__UpperCAmelCase				:						Tuple=0.0    ,				__UpperCAmelCase				:						List[Any]=0.1    ,				__UpperCAmelCase				:						Union[str, Any]=0.1    ,				__UpperCAmelCase				:						Union[str, Any]=0.02    ,				__UpperCAmelCase				:						Union[str, Any]=1e-5    ,				__UpperCAmelCase				:						List[Any]="group"    ,				__UpperCAmelCase				:						Union[str, Any]="gelu"    ,				__UpperCAmelCase				:						Any=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2)    ,				__UpperCAmelCase				:						List[str]=(5, 2, 2, 2, 2, 2, 2)    ,				__UpperCAmelCase				:						int=(1_0, 3, 3, 3, 3, 2, 2)    ,				__UpperCAmelCase				:						str=False    ,				__UpperCAmelCase				:						Any=1_2_8    ,				__UpperCAmelCase				:						str=1_6    ,				__UpperCAmelCase				:						Optional[Any]=False    ,				__UpperCAmelCase				:						Optional[int]=True    ,				__UpperCAmelCase				:						Union[str, Any]=0.05    ,				__UpperCAmelCase				:						str=1_0    ,				__UpperCAmelCase				:						Optional[Any]=2    ,				__UpperCAmelCase				:						List[Any]=0.0    ,				__UpperCAmelCase				:						Tuple=1_0    ,				__UpperCAmelCase				:						Tuple=0    ,				__UpperCAmelCase				:						Tuple=3_2_0    ,				__UpperCAmelCase				:						Any=2    ,				__UpperCAmelCase				:						Optional[int]=0.1    ,				__UpperCAmelCase				:						Any=1_0_0    ,				__UpperCAmelCase				:						str=2_5_6    ,				__UpperCAmelCase				:						Dict=2_5_6    ,				__UpperCAmelCase				:						Dict=0.1    ,				__UpperCAmelCase				:						List[str]="mean"    ,				__UpperCAmelCase				:						Optional[Any]=False    ,				__UpperCAmelCase				:						List[Any]=False    ,				__UpperCAmelCase				:						str=2_5_6    ,				__UpperCAmelCase				:						Dict=8_0    ,				__UpperCAmelCase				:						List[Any]=0    ,				__UpperCAmelCase				:						int=1    ,				__UpperCAmelCase				:						Optional[int]=2    ,				__UpperCAmelCase				:						Any=0.5    ,				**__UpperCAmelCase				:						List[str]    ,				)   ->    Tuple:
						super().__init__(**__UpperCAmelCase    ,				pad_token_id=__UpperCAmelCase    ,				bos_token_id=__UpperCAmelCase    ,				eos_token_id=__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      hidden_size
						SCREAMING_SNAKE_CASE__								=      feat_extract_norm
						SCREAMING_SNAKE_CASE__								=      feat_extract_activation
						SCREAMING_SNAKE_CASE__								=      list(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      list(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      list(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      conv_bias
						SCREAMING_SNAKE_CASE__								=      num_conv_pos_embeddings
						SCREAMING_SNAKE_CASE__								=      num_conv_pos_embedding_groups
						SCREAMING_SNAKE_CASE__								=      len(self.conv_dim       )
						SCREAMING_SNAKE_CASE__								=      num_hidden_layers
						SCREAMING_SNAKE_CASE__								=      intermediate_size
						SCREAMING_SNAKE_CASE__								=      hidden_act
						SCREAMING_SNAKE_CASE__								=      num_attention_heads
						SCREAMING_SNAKE_CASE__								=      hidden_dropout
						SCREAMING_SNAKE_CASE__								=      attention_dropout
						SCREAMING_SNAKE_CASE__								=      activation_dropout
						SCREAMING_SNAKE_CASE__								=      feat_proj_dropout
						SCREAMING_SNAKE_CASE__								=      final_dropout
						SCREAMING_SNAKE_CASE__								=      layerdrop
						SCREAMING_SNAKE_CASE__								=      layer_norm_eps
						SCREAMING_SNAKE_CASE__								=      initializer_range
						SCREAMING_SNAKE_CASE__								=      num_ctc_classes
						SCREAMING_SNAKE_CASE__								=      vocab_size
						SCREAMING_SNAKE_CASE__								=      do_stable_layer_norm
						SCREAMING_SNAKE_CASE__								=      use_weighted_layer_sum
						SCREAMING_SNAKE_CASE__								=      classifier_proj_size

						if (
						    (len(self.conv_stride       ) != self.num_feat_extract_layers)
						    or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
						    or (len(self.conv_dim       ) != self.num_feat_extract_layers)
						):
									raise ValueError(
									    """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
									    """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
									    F""" {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,"""
									    F""" `len(config.conv_kernel) = {len(self.conv_kernel       )}`."""       )

						# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
						SCREAMING_SNAKE_CASE__								=      apply_spec_augment
						SCREAMING_SNAKE_CASE__								=      mask_time_prob
						SCREAMING_SNAKE_CASE__								=      mask_time_length
						SCREAMING_SNAKE_CASE__								=      mask_time_min_masks
						SCREAMING_SNAKE_CASE__								=      mask_feature_prob
						SCREAMING_SNAKE_CASE__								=      mask_feature_length
						SCREAMING_SNAKE_CASE__								=      mask_feature_min_masks

						# parameters for pretraining with codevector quantized representations
						SCREAMING_SNAKE_CASE__								=      num_codevectors_per_group
						SCREAMING_SNAKE_CASE__								=      num_codevector_groups
						SCREAMING_SNAKE_CASE__								=      contrastive_logits_temperature
						SCREAMING_SNAKE_CASE__								=      feat_quantizer_dropout
						SCREAMING_SNAKE_CASE__								=      num_negatives
						SCREAMING_SNAKE_CASE__								=      codevector_dim
						SCREAMING_SNAKE_CASE__								=      proj_codevector_dim
						SCREAMING_SNAKE_CASE__								=      diversity_loss_weight

						# ctc loss
						SCREAMING_SNAKE_CASE__								=      ctc_loss_reduction
						SCREAMING_SNAKE_CASE__								=      ctc_zero_infinity

						# pretraining loss
						SCREAMING_SNAKE_CASE__								=      replace_prob





			@property
			def 					SCREAMING_SNAKE_CASE				(					self				:						int       )   ->    Optional[Any]:
						return functools.reduce(operator.mul    ,				self.conv_stride    ,				1       )


