


import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


_UpperCAmelCase     =logging.get_logger(__name__)

_UpperCAmelCase     ={"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCAmelCase     ={
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

_UpperCAmelCase     ={
    """roberta-base""": 512,
    """roberta-large""": 512,
    """roberta-large-mnli""": 512,
    """distilroberta-base""": 512,
    """roberta-base-openai-detector""": 512,
    """roberta-large-openai-detector""": 512,
}







class 				snake_case__(							__lowerCamelCase							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  Optional[Any]					=							VOCAB_FILES_NAMES
						SCREAMING_SNAKE_CASE__      :  Dict					=							PRETRAINED_VOCAB_FILES_MAP
						SCREAMING_SNAKE_CASE__      :  Dict					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						SCREAMING_SNAKE_CASE__      :  Tuple					=							["""input_ids""", """attention_mask"""]
						SCREAMING_SNAKE_CASE__      :  List[str]					=							RobertaTokenizer




						def __init__(				self   ,						__lowercase=None   ,						__lowercase=None   ,						__lowercase=None   ,						__lowercase="replace"   ,						__lowercase="<s>"   ,						__lowercase="</s>"   ,						__lowercase="</s>"   ,						__lowercase="<s>"   ,						__lowercase="<unk>"   ,						__lowercase="<pad>"   ,						__lowercase="<mask>"   ,						__lowercase=False   ,						__lowercase=True   ,						**__lowercase   ,						)					->    Optional[Any]:
										super().__init__(
										    UpperCAmelCase_   ,						UpperCAmelCase_   ,						tokenizer_file=UpperCAmelCase_   ,						errors=UpperCAmelCase_   ,						bos_token=UpperCAmelCase_   ,						eos_token=UpperCAmelCase_   ,						sep_token=UpperCAmelCase_   ,						cls_token=UpperCAmelCase_   ,						unk_token=UpperCAmelCase_   ,						pad_token=UpperCAmelCase_   ,						mask_token=UpperCAmelCase_   ,						add_prefix_space=UpperCAmelCase_   ,						trim_offsets=UpperCAmelCase_   ,						**UpperCAmelCase_   ,						)

										lowerCAmelCase_ :      Optional[Any]    				=				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
										if pre_tok_state.get('''add_prefix_space'''   ,						UpperCAmelCase_							) != add_prefix_space:
														lowerCAmelCase_ :      Tuple    				=				getattr(UpperCAmelCase_   ,						pre_tok_state.pop('''type'''							)							)
														lowerCAmelCase_ :      Optional[Any]    				=				add_prefix_space
														lowerCAmelCase_ :      List[str]    				=				pre_tok_class(**UpperCAmelCase_							)

										lowerCAmelCase_ :      str    				=				add_prefix_space

										lowerCAmelCase_ :      Optional[int]    				=				'post_processor'
										lowerCAmelCase_ :      Dict    				=				getattr(self.backend_tokenizer   ,						UpperCAmelCase_   ,						UpperCAmelCase_							)
										if tokenizer_component_instance:
														lowerCAmelCase_ :      Tuple    				=				json.loads(tokenizer_component_instance.__getstate__()							)

														# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
														if "sep" in state:
																		lowerCAmelCase_ :      Optional[int]    				=				tuple(state['''sep''']							)
														if "cls" in state:
																		lowerCAmelCase_ :      Tuple    				=				tuple(state['''cls''']							)

														lowerCAmelCase_ :      Union[str, Any]    				=				False

														if state.get('''add_prefix_space'''   ,						UpperCAmelCase_							) != add_prefix_space:
																		lowerCAmelCase_ :      int    				=				add_prefix_space
																		lowerCAmelCase_ :      List[str]    				=				True

														if state.get('''trim_offsets'''   ,						UpperCAmelCase_							) != trim_offsets:
																		lowerCAmelCase_ :      Tuple    				=				trim_offsets
																		lowerCAmelCase_ :      List[str]    				=				True

														if changes_to_apply:
																		lowerCAmelCase_ :      Tuple    				=				getattr(UpperCAmelCase_   ,						state.pop('''type'''							)							)
																		lowerCAmelCase_ :      Any    				=				component_class(**UpperCAmelCase_							)
																		setattr(self.backend_tokenizer   ,						UpperCAmelCase_   ,						UpperCAmelCase_							)




						@property
						def 						lowercase_ (				self							)					->    Union[str, Any]:
										if self._mask_token is None:
														if self.verbose:
																		logger.error('''Using mask_token, but it is not set yet.'''							)
														return None
										return str(self._mask_token							)




						@mask_token.setter
						def 						lowercase_ (				self   ,						__lowercase							)					->    Union[str, Any]:
										lowerCAmelCase_ :      str    				=				AddedToken(UpperCAmelCase_   ,						lstrip=UpperCAmelCase_   ,						rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_   ,						UpperCAmelCase_							) else value
										lowerCAmelCase_ :      Any    				=				value




						def 						lowercase_ (				self   ,						*__lowercase   ,						**__lowercase							)					->    str:
										lowerCAmelCase_ :      str    				=				kwargs.get('''is_split_into_words'''   ,						UpperCAmelCase_							)
										assert self.add_prefix_space or not is_split_into_words, (
										    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*UpperCAmelCase_   ,						**UpperCAmelCase_							)




						def 						lowercase_ (				self   ,						*__lowercase   ,						**__lowercase							)					->    Optional[int]:
										lowerCAmelCase_ :      Dict    				=				kwargs.get('''is_split_into_words'''   ,						UpperCAmelCase_							)

										assert self.add_prefix_space or not is_split_into_words, (
										    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*UpperCAmelCase_   ,						**UpperCAmelCase_							)




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    Optional[Any]:
										lowerCAmelCase_ :      Tuple    				=				self._tokenizer.model.save(UpperCAmelCase_   ,						name=UpperCAmelCase_							)
										return tuple(UpperCAmelCase_							)




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase=None							)					->    List[Any]:
										lowerCAmelCase_ :      Union[str, Any]    				=				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
										if token_ids_a is None:
														return output

										return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    Dict:
										lowerCAmelCase_ :      Any    				=				[self.sep_token_id]
										lowerCAmelCase_ :      str    				=				[self.cls_token_id]

										if token_ids_a is None:
														return len(cls + token_ids_a + sep							) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCAmelCase			:  Optional[int]     =logging.get_logger(__name__)

_UpperCAmelCase			:  Union[str, Any]     ={
    """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json""",
}







class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  str					=							"""gpt_neox_japanese"""




						def __init__(				self   ,						__lowercase=3_2_0_0_0   ,						__lowercase=2_5_6_0   ,						__lowercase=3_2   ,						__lowercase=3_2   ,						__lowercase=4   ,						__lowercase="gelu"   ,						__lowercase=1.00   ,						__lowercase=1_0_0_0_0   ,						__lowercase=2_0_4_8   ,						__lowercase=0.02   ,						__lowercase=1e-5   ,						__lowercase=True   ,						__lowercase=3_1_9_9_6   ,						__lowercase=3_1_9_9_9   ,						__lowercase=0.1   ,						__lowercase=0.0   ,						**__lowercase   ,						)					->    str:
										super().__init__(bos_token_id=__lowercase   ,						eos_token_id=__lowercase   ,						**__lowercase							)
										lowerCAmelCase_ :      Optional[Any]    				=				vocab_size
										lowerCAmelCase_ :      Tuple    				=				max_position_embeddings
										lowerCAmelCase_ :      Optional[Any]    				=				hidden_size
										lowerCAmelCase_ :      Optional[Any]    				=				num_hidden_layers
										lowerCAmelCase_ :      str    				=				num_attention_heads
										lowerCAmelCase_ :      str    				=				intermediate_multiple_size
										lowerCAmelCase_ :      str    				=				hidden_act
										lowerCAmelCase_ :      Dict    				=				rotary_pct
										lowerCAmelCase_ :      Union[str, Any]    				=				rotary_emb_base
										lowerCAmelCase_ :      int    				=				initializer_range
										lowerCAmelCase_ :      Any    				=				layer_norm_eps
										lowerCAmelCase_ :      Optional[Any]    				=				use_cache
										lowerCAmelCase_ :      Tuple    				=				attention_dropout
										lowerCAmelCase_ :      Dict    				=				hidden_dropout