


'''simple docstring'''




import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD



class  lowerCAmelCase				(				nn.Module							):
	_lowerCamelCase   :					int
	_lowerCamelCase   :					int
	_lowerCamelCase   :					float			      = 0.0
	_lowerCamelCase   :					int			      = 1
	_lowerCamelCase   :					int			      = 1
	_lowerCamelCase   :					bool			      = True
	_lowerCamelCase   :					bool			      = False
	_lowerCamelCase   :					bool			      = False
	_lowerCamelCase   :					bool			      = False
	_lowerCamelCase   :					jnp.dtype			      = jnp.floataa



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Tuple      							= []
		lowerCAmelCase					:    Union[str, Any]      							= []

		for i in range(self.num_layers    ):
			lowerCAmelCase					:    Optional[Any]      							= self.in_channels if i == 0 else self.out_channels

			lowerCAmelCase					:    Optional[int]      							= FlaxResnetBlockaD(
			    in_channels=snake_case__ ,					out_channels=self.out_channels ,					dropout_prob=self.dropout ,					dtype=self.dtype ,					)
			resnets.append(snake_case__    )

			lowerCAmelCase					:    List[str]      							= FlaxTransformeraDModel(
			    in_channels=self.out_channels ,					n_heads=self.num_attention_heads ,					d_head=self.out_channels // self.num_attention_heads ,					depth=1 ,					use_linear_projection=self.use_linear_projection ,					only_cross_attention=self.only_cross_attention ,					use_memory_efficient_attention=self.use_memory_efficient_attention ,					dtype=self.dtype ,					)
			attentions.append(snake_case__    )

		lowerCAmelCase					:    List[Any]      							= resnets
		lowerCAmelCase					:    int      							= attentions

		if self.add_downsample:
			lowerCAmelCase					:    Optional[Any]      							= FlaxDownsampleaD(self.out_channels ,					dtype=self.dtype    )



	def __call__(					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__=True    ):
		lowerCAmelCase					:    List[Any]      							= ()

		for resnet, attn in zip(self.resnets ,					self.attentions    ):
			lowerCAmelCase					:    Optional[Any]      							= resnet(snake_case__ ,					snake_case__ ,					deterministic=snake_case__    )
			lowerCAmelCase					:    int      							= attn(snake_case__ ,					snake_case__ ,					deterministic=snake_case__    )
			output_states += (hidden_states,)

		if self.add_downsample:
			lowerCAmelCase					:    Optional[int]      							= self.downsamplers_a(snake_case__    )
			output_states += (hidden_states,)

		return hidden_states, output_states



class  lowerCAmelCase				(				nn.Module							):
	_lowerCamelCase   :					int
	_lowerCamelCase   :					int
	_lowerCamelCase   :					float			      = 0.0
	_lowerCamelCase   :					int			      = 1
	_lowerCamelCase   :					bool			      = True
	_lowerCamelCase   :					jnp.dtype			      = jnp.floataa



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[int]      							= []

		for i in range(self.num_layers    ):
			lowerCAmelCase					:    Union[str, Any]      							= self.in_channels if i == 0 else self.out_channels

			lowerCAmelCase					:    Tuple      							= FlaxResnetBlockaD(
			    in_channels=snake_case__ ,					out_channels=self.out_channels ,					dropout_prob=self.dropout ,					dtype=self.dtype ,					)
			resnets.append(snake_case__    )
		lowerCAmelCase					:    Optional[int]      							= resnets

		if self.add_downsample:
			lowerCAmelCase					:    str      							= FlaxDownsampleaD(self.out_channels ,					dtype=self.dtype    )



	def __call__(					self ,					snake_case__ ,					snake_case__ ,					snake_case__=True    ):
		lowerCAmelCase					:    List[Any]      							= ()

		for resnet in self.resnets:
			lowerCAmelCase					:    List[str]      							= resnet(snake_case__ ,					snake_case__ ,					deterministic=snake_case__    )
			output_states += (hidden_states,)

		if self.add_downsample:
			lowerCAmelCase					:    Tuple      							= self.downsamplers_a(snake_case__    )
			output_states += (hidden_states,)

		return hidden_states, output_states



class  lowerCAmelCase				(				nn.Module							):
	_lowerCamelCase   :					int
	_lowerCamelCase   :					int
	_lowerCamelCase   :					int
	_lowerCamelCase   :					float			      = 0.0
	_lowerCamelCase   :					int			      = 1
	_lowerCamelCase   :					int			      = 1
	_lowerCamelCase   :					bool			      = True
	_lowerCamelCase   :					bool			      = False
	_lowerCamelCase   :					bool			      = False
	_lowerCamelCase   :					bool			      = False
	_lowerCamelCase   :					jnp.dtype			      = jnp.floataa



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[int]      							= []
		lowerCAmelCase					:    Tuple      							= []

		for i in range(self.num_layers    ):
			lowerCAmelCase					:    Optional[Any]      							= self.in_channels if (i == self.num_layers - 1) else self.out_channels
			lowerCAmelCase					:    str      							= self.prev_output_channel if i == 0 else self.out_channels

			lowerCAmelCase					:    Optional[Any]      							= FlaxResnetBlockaD(
			    in_channels=resnet_in_channels + res_skip_channels ,					out_channels=self.out_channels ,					dropout_prob=self.dropout ,					dtype=self.dtype ,					)
			resnets.append(snake_case__    )

			lowerCAmelCase					:    List[Any]      							= FlaxTransformeraDModel(
			    in_channels=self.out_channels ,					n_heads=self.num_attention_heads ,					d_head=self.out_channels // self.num_attention_heads ,					depth=1 ,					use_linear_projection=self.use_linear_projection ,					only_cross_attention=self.only_cross_attention ,					use_memory_efficient_attention=self.use_memory_efficient_attention ,					dtype=self.dtype ,					)
			attentions.append(snake_case__    )

		lowerCAmelCase					:    List[str]      							= resnets
		lowerCAmelCase					:    str      							= attentions

		if self.add_upsample:
			lowerCAmelCase					:    List[str]      							= FlaxUpsampleaD(self.out_channels ,					dtype=self.dtype    )



	def __call__(					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__=True    ):
		for resnet, attn in zip(self.resnets ,					self.attentions    ):
			# pop res hidden states
			lowerCAmelCase					:    Tuple      							= res_hidden_states_tuple[-1]
			lowerCAmelCase					:    Union[str, Any]      							= res_hidden_states_tuple[:-1]
			lowerCAmelCase					:    Dict      							= jnp.concatenate((hidden_states, res_hidden_states) ,					axis=-1    )

			lowerCAmelCase					:    str      							= resnet(snake_case__ ,					snake_case__ ,					deterministic=snake_case__    )
			lowerCAmelCase					:    Tuple      							= attn(snake_case__ ,					snake_case__ ,					deterministic=snake_case__    )

		if self.add_upsample:
			lowerCAmelCase					:    List[Any]      							= self.upsamplers_a(snake_case__    )

		return hidden_states



class  lowerCAmelCase				(				nn.Module							):
	_lowerCamelCase   :					int
	_lowerCamelCase   :					int
	_lowerCamelCase   :					int
	_lowerCamelCase   :					float			      = 0.0
	_lowerCamelCase   :					int			      = 1
	_lowerCamelCase   :					bool			      = True
	_lowerCamelCase   :					jnp.dtype			      = jnp.floataa



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    str      							= []

		for i in range(self.num_layers    ):
			lowerCAmelCase					:    Union[str, Any]      							= self.in_channels if (i == self.num_layers - 1) else self.out_channels
			lowerCAmelCase					:    str      							= self.prev_output_channel if i == 0 else self.out_channels

			lowerCAmelCase					:    Dict      							= FlaxResnetBlockaD(
			    in_channels=resnet_in_channels + res_skip_channels ,					out_channels=self.out_channels ,					dropout_prob=self.dropout ,					dtype=self.dtype ,					)
			resnets.append(snake_case__    )

		lowerCAmelCase					:    Tuple      							= resnets

		if self.add_upsample:
			lowerCAmelCase					:    Dict      							= FlaxUpsampleaD(self.out_channels ,					dtype=self.dtype    )



	def __call__(					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__=True    ):
		for resnet in self.resnets:
			# pop res hidden states
			lowerCAmelCase					:    Tuple      							= res_hidden_states_tuple[-1]
			lowerCAmelCase					:    Union[str, Any]      							= res_hidden_states_tuple[:-1]
			lowerCAmelCase					:    Dict      							= jnp.concatenate((hidden_states, res_hidden_states) ,					axis=-1    )

			lowerCAmelCase					:    Dict      							= resnet(snake_case__ ,					snake_case__ ,					deterministic=snake_case__    )

		if self.add_upsample:
			lowerCAmelCase					:    Tuple      							= self.upsamplers_a(snake_case__    )

		return hidden_states



class  lowerCAmelCase				(				nn.Module							):
	_lowerCamelCase   :					int
	_lowerCamelCase   :					float			      = 0.0
	_lowerCamelCase   :					int			      = 1
	_lowerCamelCase   :					int			      = 1
	_lowerCamelCase   :					bool			      = False
	_lowerCamelCase   :					bool			      = False
	_lowerCamelCase   :					jnp.dtype			      = jnp.floataa



	def 				lowercase     (					self    ):
		# there is always at least one resnet
		lowerCAmelCase					:    Tuple      							= [
		    FlaxResnetBlockaD(
		        in_channels=self.in_channels ,					out_channels=self.in_channels ,					dropout_prob=self.dropout ,					dtype=self.dtype ,					)
		]

		lowerCAmelCase					:    Union[str, Any]      							= []

		for _ in range(self.num_layers    ):
			lowerCAmelCase					:    Optional[int]      							= FlaxTransformeraDModel(
			    in_channels=self.in_channels ,					n_heads=self.num_attention_heads ,					d_head=self.in_channels // self.num_attention_heads ,					depth=1 ,					use_linear_projection=self.use_linear_projection ,					use_memory_efficient_attention=self.use_memory_efficient_attention ,					dtype=self.dtype ,					)
			attentions.append(snake_case__    )

			lowerCAmelCase					:    Tuple      							= FlaxResnetBlockaD(
			    in_channels=self.in_channels ,					out_channels=self.in_channels ,					dropout_prob=self.dropout ,					dtype=self.dtype ,					)
			resnets.append(snake_case__    )

		lowerCAmelCase					:    int      							= resnets
		lowerCAmelCase					:    Optional[Any]      							= attentions



	def __call__(					self ,					snake_case__ ,					snake_case__ ,					snake_case__ ,					snake_case__=True    ):
		lowerCAmelCase					:    List[str]      							= self.resnets[0](snake_case__ ,					snake_case__    )
		for attn, resnet in zip(self.attentions ,					self.resnets[1:]    ):
			lowerCAmelCase					:    Tuple      							= attn(snake_case__ ,					snake_case__ ,					deterministic=snake_case__    )
			lowerCAmelCase					:    Union[str, Any]      							= resnet(snake_case__ ,					snake_case__ ,					deterministic=snake_case__    )

		return hidden_states




'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCAmelCase  :		str		     =	logging.get_logger(__name__)

_lowerCAmelCase  :		Tuple		     =	{
    's-JoL/Open-Llama-V1': 'https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json',
}



class  lowerCAmelCase				(				a							):
	_lowerCamelCase   :					Union[str, Any]			      = """open-llama"""



	def __init__(					self ,					snake_case__=10_0000 ,					snake_case__=4096 ,					snake_case__=1_1008 ,					snake_case__=32 ,					snake_case__=32 ,					snake_case__="silu" ,					snake_case__=2048 ,					snake_case__=0.0_2 ,					snake_case__=1e-6 ,					snake_case__=True ,					snake_case__=0 ,					snake_case__=1 ,					snake_case__=2 ,					snake_case__=False ,					snake_case__=True ,					snake_case__=0.1 ,					snake_case__=0.1 ,					snake_case__=True ,					snake_case__=True ,					snake_case__=None ,					**snake_case__ ,					):
		lowerCAmelCase					:    Tuple      							= vocab_size
		lowerCAmelCase					:    Optional[Any]      							= max_position_embeddings
		lowerCAmelCase					:    List[Any]      							= hidden_size
		lowerCAmelCase					:    List[Any]      							= intermediate_size
		lowerCAmelCase					:    Tuple      							= num_hidden_layers
		lowerCAmelCase					:    List[Any]      							= num_attention_heads
		lowerCAmelCase					:    List[Any]      							= hidden_act
		lowerCAmelCase					:    Union[str, Any]      							= initializer_range
		lowerCAmelCase					:    str      							= rms_norm_eps
		lowerCAmelCase					:    Optional[int]      							= use_cache
		lowerCAmelCase					:    Dict      							= kwargs.pop(
		    'use_memorry_efficient_attention' ,					snake_case__    )
		lowerCAmelCase					:    Optional[int]      							= hidden_dropout_prob
		lowerCAmelCase					:    Optional[Any]      							= attention_dropout_prob
		lowerCAmelCase					:    Union[str, Any]      							= use_stable_embedding
		lowerCAmelCase					:    Tuple      							= shared_input_output_embedding
		lowerCAmelCase					:    Tuple      							= rope_scaling
		self._rope_scaling_validation()

		super().__init__(
		    pad_token_id=snake_case__ ,					bos_token_id=snake_case__ ,					eos_token_id=snake_case__ ,					tie_word_embeddings=snake_case__ ,					**snake_case__ ,					)



	def 				lowercase     (					self    ):
		if self.rope_scaling is None:
			return

		if not isinstance(self.rope_scaling ,					snake_case__    ) or len(self.rope_scaling    ) != 2:
			raise ValueError(
			    '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
			    f"got {self.rope_scaling}"    )
		lowerCAmelCase					:    List[Any]      							= self.rope_scaling.get('type' ,					snake_case__    )
		lowerCAmelCase					:    List[str]      							= self.rope_scaling.get('factor' ,					snake_case__    )
		if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
			raise ValueError(
			    f"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"    )
		if rope_scaling_factor is None or not isinstance(snake_case__ ,					snake_case__    ) or rope_scaling_factor <= 1.0:
			raise ValueError(f"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"    )
