
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_						: Optional[Any]         =  logging.get_logger(__name__)

lowerCamelCase_						: List[Any]         =  {
    """s-JoL/Open-Llama-V1""": """https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json""",
}

class        _UpperCamelCase				(					__lowerCamelCase  ):






   '''simple docstring'''

   __UpperCamelCase      :						str					      = """open-llama"""

   def __init__(       self :       Optional[Any]						,	snake_case_ :       Union[str, Any]=10_0000						,	snake_case_ :       Tuple=4096						,	snake_case_ :       List[Any]=1_1008						,	snake_case_ :       Any=32						,	snake_case_ :       List[str]=32						,	snake_case_ :       int="silu"						,	snake_case_ :       int=2048						,	snake_case_ :       List[str]=0.02						,	snake_case_ :       int=1e-6						,	snake_case_ :       List[Any]=True						,	snake_case_ :       str=0						,	snake_case_ :       Any=1						,	snake_case_ :       str=2						,	snake_case_ :       List[str]=False						,	snake_case_ :       int=True						,	snake_case_ :       Optional[int]=0.1						,	snake_case_ :       Optional[Any]=0.1						,	snake_case_ :       Optional[int]=True						,	snake_case_ :       str=True						,	snake_case_ :       Union[str, Any]=None						,	**snake_case_ :       List[Any]						,	):
       UpperCamelCase_:							str	=						vocab_size
       UpperCamelCase_:							List[Any]	=						max_position_embeddings
       UpperCamelCase_:							Dict	=						hidden_size
       UpperCamelCase_:							List[Any]	=						intermediate_size
       UpperCamelCase_:							Optional[int]	=						num_hidden_layers
       UpperCamelCase_:							Union[str, Any]	=						num_attention_heads
       UpperCamelCase_:							Optional[int]	=						hidden_act
       UpperCamelCase_:							int	=						initializer_range
       UpperCamelCase_:							Optional[int]	=						rms_norm_eps
       UpperCamelCase_:							Optional[Any]	=						use_cache
       UpperCamelCase_:							Dict	=						kwargs.pop(
           """use_memorry_efficient_attention"""						,	SCREAMING_SNAKE_CASE_      )
       UpperCamelCase_:							Dict	=						hidden_dropout_prob
       UpperCamelCase_:							Optional[Any]	=						attention_dropout_prob
       UpperCamelCase_:							int	=						use_stable_embedding
       UpperCamelCase_:							Tuple	=						shared_input_output_embedding
       UpperCamelCase_:							Optional[Any]	=						rope_scaling
       self._rope_scaling_validation()

       super().__init__(
           pad_token_id=SCREAMING_SNAKE_CASE_						,	bos_token_id=SCREAMING_SNAKE_CASE_						,	eos_token_id=SCREAMING_SNAKE_CASE_						,	tie_word_embeddings=SCREAMING_SNAKE_CASE_						,	**SCREAMING_SNAKE_CASE_						,	)




   def       lowerCAmelCase__	(       self :       Any      ):
       if self.rope_scaling is None:
           return

       if not isinstance(self.rope_scaling						,	SCREAMING_SNAKE_CASE_      ) or len(self.rope_scaling      ) != 2:
           raise ValueError(
               """`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, """
               f'''got {self.rope_scaling}'''      )
       UpperCamelCase_:							Any	=						self.rope_scaling.get("""type"""						,	SCREAMING_SNAKE_CASE_      )
       UpperCamelCase_:							str	=						self.rope_scaling.get("""factor"""						,	SCREAMING_SNAKE_CASE_      )
       if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
           raise ValueError(
               f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''      )
       if rope_scaling_factor is None or not isinstance(SCREAMING_SNAKE_CASE_						,	SCREAMING_SNAKE_CASE_      ) or rope_scaling_factor <= 1.0:
           raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''      )








from manim import *

class        _UpperCamelCase				(					_A  ):






			'''simple docstring'''

			def       lowerCAmelCase__	(       self :       int      ):
							UpperCamelCase_:							Dict	=						Rectangle(height=0.5						,	width=0.5      )
							UpperCamelCase_:							Dict	=						Rectangle(height=0.46						,	width=0.46      ).set_stroke(width=0      )

							UpperCamelCase_:							Tuple	=						[mem.copy() for i in range(6      )]
							UpperCamelCase_:							List[str]	=						[mem.copy() for i in range(6      )]
							UpperCamelCase_:							List[str]	=						VGroup(*snake_case_      ).arrange(snake_case_						,	buff=0      )
							UpperCamelCase_:							Tuple	=						VGroup(*snake_case_      ).arrange(snake_case_						,	buff=0      )
							UpperCamelCase_:							Union[str, Any]	=						VGroup(snake_case_						,	snake_case_      ).arrange(snake_case_						,	buff=0      )
							UpperCamelCase_:							Optional[Any]	=						Text("""CPU"""						,	font_size=24      )
							UpperCamelCase_:							int	=						Group(snake_case_						,	snake_case_      ).arrange(snake_case_						,	buff=0.5						,	aligned_edge=snake_case_      )
							cpu.move_to([-2.5, -0.5, 0]      )
							self.add(snake_case_      )

							UpperCamelCase_:							Optional[int]	=						[mem.copy() for i in range(1      )]
							UpperCamelCase_:							Dict	=						VGroup(*snake_case_      ).arrange(snake_case_						,	buff=0      )
							UpperCamelCase_:							Optional[int]	=						Text("""GPU"""						,	font_size=24      )
							UpperCamelCase_:							Optional[int]	=						Group(snake_case_						,	snake_case_      ).arrange(snake_case_						,	buff=0.5						,	aligned_edge=snake_case_      )
							gpu.align_to(snake_case_						,	snake_case_      )
							gpu.set_x(gpu.get_x() - 1      )

							self.add(snake_case_      )

							UpperCamelCase_:							Dict	=						[mem.copy() for i in range(6      )]
							UpperCamelCase_:							List[str]	=						VGroup(*snake_case_      ).arrange(snake_case_						,	buff=0      )

							UpperCamelCase_:							Any	=						Text("""Model"""						,	font_size=24      )
							UpperCamelCase_:							Optional[Any]	=						Group(snake_case_						,	snake_case_      ).arrange(snake_case_						,	buff=0.5						,	aligned_edge=snake_case_      )
							model.move_to([3, -1.0, 0]      )

							self.play(
							    Create(snake_case_						,	run_time=1      )						,	Create(snake_case_						,	run_time=1      )						,	Create(snake_case_						,	run_time=1      )						,	)

							UpperCamelCase_:							List[Any]	=						MarkupText(
							    f'''First, an empty model skeleton is loaded\ninto <span fgcolor=\'{YELLOW}\'>memory</span> without using much RAM.'''						,	font_size=24						,	)

							UpperCamelCase_:							Optional[Any]	=						Square(side_length=2.2      )
							key.move_to([-5, 2, 0]      )

							UpperCamelCase_:							Union[str, Any]	=						MarkupText(
							    f'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''						,	font_size=18						,	)

							key_text.move_to([-5, 2.4, 0]      )

							step_a.move_to([2, 2, 0]      )
							self.play(Write(snake_case_						,	run_time=2.5      )						,	Write(snake_case_      )						,	Write(snake_case_      )      )

							self.add(snake_case_      )

							UpperCamelCase_:							Union[str, Any]	=						[]
							UpperCamelCase_:							Union[str, Any]	=						[]
							UpperCamelCase_:							Tuple	=						[]
							for i, rect in enumerate(snake_case_      ):
											UpperCamelCase_:							Tuple	=						Rectangle(height=0.46						,	width=0.46      ).set_stroke(width=0.0      ).set_fill(snake_case_						,	opacity=0.7      )
											cpu_target.move_to(snake_case_      )
											cpu_target.generate_target()
											UpperCamelCase_:							int	=						0.46 / 4
											UpperCamelCase_:							Optional[int]	=						0.46 / 3

											if i == 0:
															cpu_target.target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT      )						,	buff=0.02						,	direction=snake_case_      )
															cpu_target.target.set_x(cpu_target.target.get_x() + 0.1      )
											elif i == 3:
															cpu_target.target.next_to(cpu_targs[0].target						,	direction=snake_case_						,	buff=0.0      )
											else:
															cpu_target.target.next_to(cpu_targs[i - 1].target						,	direction=snake_case_						,	buff=0.0      )
											cpu_targs.append(snake_case_      )

											first_animations.append(rect.animate(run_time=0.5      ).set_stroke(snake_case_      )      )
											second_animations.append(MoveToTarget(snake_case_						,	run_time=1.5      )      )

							self.play(*snake_case_      )
							self.play(*snake_case_      )

							self.wait()






