






'''simple docstring'''







from typing import TYPE_CHECKING

from ...file_utils import _LazyModule, is_torch_available
from ...utils import OptionalDependencyNotAvailable


__lowerCAmelCase       		=      {
    'configuration_gpt_neox_japanese': ['GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP', 'GPTNeoXJapaneseConfig'],
    'tokenization_gpt_neox_japanese': ['GPTNeoXJapaneseTokenizer'],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__lowerCAmelCase       		=      [
		    'GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'GPTNeoXJapaneseForCausalLM',
		    'GPTNeoXJapaneseLayer',
		    'GPTNeoXJapaneseModel',
		    'GPTNeoXJapanesePreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_gpt_neox_japanese import GPT_NEOX_JAPANESE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTNeoXJapaneseConfig
		from .tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_gpt_neox_japanese import (
				    GPT_NEOX_JAPANESE_PRETRAINED_MODEL_ARCHIVE_LIST,
				    GPTNeoXJapaneseForCausalLM,
				    GPTNeoXJapaneseLayer,
				    GPTNeoXJapaneseModel,
				    GPTNeoXJapanesePreTrainedModel,
				)


else:
		import sys

		__lowerCAmelCase       		=      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)












'''simple docstring'''







from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional

from packaging import version


if TYPE_CHECKING:
		from ... import PreTrainedTokenizer, TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import is_torch_available, logging


__lowerCAmelCase       		=      logging.get_logger(__name__)

__lowerCAmelCase       		=      {
    'bigscience/bloom': 'https://huggingface.co/bigscience/bloom/resolve/main/config.json',
    'bigscience/bloom-560m': 'https://huggingface.co/bigscience/bloom-560m/blob/main/config.json',
    'bigscience/bloom-1b1': 'https://huggingface.co/bigscience/bloom-1b1/blob/main/config.json',
    'bigscience/bloom-1b7': 'https://huggingface.co/bigscience/bloom-1b7/blob/main/config.json',
    'bigscience/bloom-3b': 'https://huggingface.co/bigscience/bloom-3b/blob/main/config.json',
    'bigscience/bloom-7b1': 'https://huggingface.co/bigscience/bloom-7b1/blob/main/config.json',
}





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				"bloom"
			snake_case__  =				["past_key_values"]
			snake_case__  =				{
			    "num_hidden_layers": "n_layer",
			    "num_attention_heads": "n_head",
			}






			def __init__( self      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			int=25_0880  ,						__SCREAMING_SNAKE_CASE      :			Dict=64  ,						__SCREAMING_SNAKE_CASE      :			Tuple=2  ,						__SCREAMING_SNAKE_CASE      :			int=8  ,						__SCREAMING_SNAKE_CASE      :			Any=1e-5  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=0.02  ,						__SCREAMING_SNAKE_CASE      :			Union[str, Any]=True  ,						__SCREAMING_SNAKE_CASE      :			int=1  ,						__SCREAMING_SNAKE_CASE      :			Any=2  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=False  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=0.0  ,						__SCREAMING_SNAKE_CASE      :			str=0.0  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=1  ,						__SCREAMING_SNAKE_CASE      :			List[str]=False  ,						**__SCREAMING_SNAKE_CASE      :			str  ,						)			->			Any:
							a_		:  Optional[int]											=      vocab_size
							# Backward compatibility with n_embed kwarg
							a_		:  Any											=      kwargs.pop('''n_embed'''  ,						__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[int]											=      hidden_size if n_embed is None else n_embed
							a_		:  int											=      n_layer
							a_		:  str											=      n_head
							a_		:  Optional[int]											=      layer_norm_epsilon
							a_		:  Dict											=      initializer_range
							a_		:  List[str]											=      use_cache
							a_		:  Dict											=      pretraining_tp
							a_		:  Optional[Any]											=      apply_residual_connection_post_layernorm
							a_		:  Optional[Any]											=      hidden_dropout
							a_		:  List[str]											=      attention_dropout

							a_		:  Dict											=      bos_token_id
							a_		:  Optional[int]											=      eos_token_id
							a_		:  Any											=      slow_but_exact

							super().__init__(bos_token_id=__SCREAMING_SNAKE_CASE  ,						eos_token_id=__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				version.parse("1.12"       )






			def __init__( self      :			Tuple  ,						__SCREAMING_SNAKE_CASE      :			PretrainedConfig  ,						__SCREAMING_SNAKE_CASE      :			str = "default"  ,						__SCREAMING_SNAKE_CASE      :			List[PatchingSpec] = None  ,						__SCREAMING_SNAKE_CASE      :			bool = False  ,						)			->			Optional[Any]:
							super().__init__(__SCREAMING_SNAKE_CASE  ,						task=__SCREAMING_SNAKE_CASE  ,						patching_specs=__SCREAMING_SNAKE_CASE  ,						use_past=__SCREAMING_SNAKE_CASE					)
							if not getattr(self._config  ,						'''pad_token_id'''  ,						__SCREAMING_SNAKE_CASE					):
											# TODO: how to do that better?
											a_		:  Tuple											=      0






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Tuple					)			->			Mapping[str, Mapping[int, str]]:
							a_		:  Optional[Any]											=      OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}					)
							if self.use_past:
											# BLOOM stores values on dynamic axis 2. For more details see: https://github.com/huggingface/transformers/pull/18344
											self.fill_with_past_key_values_(__SCREAMING_SNAKE_CASE  ,						direction='''inputs'''  ,						inverted_values_shape=__SCREAMING_SNAKE_CASE					)
											a_		:  Union[str, Any]											=      {0: '''batch''', 1: '''past_sequence + sequence'''}
							else:
											a_		:  Union[str, Any]											=      {0: '''batch''', 1: '''sequence'''}

							return common_inputs






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			int:
							return self._config.n_layer






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			int:
							return self._config.n_head






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			float:
							return 1e-3






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			"PreTrainedTokenizer"  ,						__SCREAMING_SNAKE_CASE      :			int = -1  ,						__SCREAMING_SNAKE_CASE      :			int = -1  ,						__SCREAMING_SNAKE_CASE      :			bool = False  ,						__SCREAMING_SNAKE_CASE      :			Optional["TensorType"] = None  ,						)			->			Mapping[str, Any]:
							a_		:  Dict											=      super(__SCREAMING_SNAKE_CASE  ,						self					).generate_dummy_inputs(
							    __SCREAMING_SNAKE_CASE  ,						batch_size=__SCREAMING_SNAKE_CASE  ,						seq_length=__SCREAMING_SNAKE_CASE  ,						is_pair=__SCREAMING_SNAKE_CASE  ,						framework=__SCREAMING_SNAKE_CASE					)

							# We need to order the input in the way they appears in the forward()
							a_		:  Union[str, Any]											=      OrderedDict({'''input_ids''': common_inputs['''input_ids''']}					)

							# Need to add the past_keys
							if self.use_past:
											if not is_torch_available():
															raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''					)
											else:
															import torch

															a_				,			a_		:  Any											=      common_inputs['''input_ids'''].shape
															# Not using the same length for past_key_values
															a_		:  str											=      seqlen + 2
															a_		:  Any											=      self._config.hidden_size // self.num_attention_heads
															a_		:  Optional[int]											=      (
															    batch * self.num_attention_heads,
															    head_dim,
															    past_key_values_length,
															)
															a_		:  Any											=      (
															    batch * self.num_attention_heads,
															    past_key_values_length,
															    head_dim,
															)
															a_		:  List[str]											=      [
															    (torch.zeros(__SCREAMING_SNAKE_CASE					), torch.zeros(__SCREAMING_SNAKE_CASE					)) for _ in range(self.num_layers					)
															]

							a_		:  Union[str, Any]											=      common_inputs['''attention_mask''']
							if self.use_past:
											a_		:  Optional[int]											=      ordered_inputs['''attention_mask'''].dtype
											a_		:  List[Any]											=      torch.cat(
											    [ordered_inputs['''attention_mask'''], torch.ones(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE  ,						dtype=__SCREAMING_SNAKE_CASE					)]  ,						dim=1					)

							return ordered_inputs







			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]					)			->			int:
							return 13




