






'''simple docstring'''
from manim import *






class      __SCREAMING_SNAKE_CASE      (			lowerCamelCase				):




	def      __magic_name__	(      self	:							int      )	->						Any:
							SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=Rectangle(height=0.5					,      width=0.5      )
							SCREAMING_SNAKE_CASE__     :				int									=Rectangle(height=0.46					,      width=0.46      ).set_stroke(width=0      )

							SCREAMING_SNAKE_CASE__     :				Optional[int]									=Rectangle(height=0.25					,      width=0.25      )

							SCREAMING_SNAKE_CASE__     :				int									=[mem.copy() for i in range(6      )]
							SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=[mem.copy() for i in range(6      )]
							SCREAMING_SNAKE_CASE__     :				Tuple									=VGroup(*__lowercase      ).arrange(__lowercase					,      buff=0      )
							SCREAMING_SNAKE_CASE__     :				List[Any]									=VGroup(*__lowercase      ).arrange(__lowercase					,      buff=0      )
							SCREAMING_SNAKE_CASE__     :				int									=VGroup(__lowercase					,      __lowercase      ).arrange(__lowercase					,      buff=0      )
							SCREAMING_SNAKE_CASE__     :				str									=Text('''CPU'''					,      font_size=24      )
							SCREAMING_SNAKE_CASE__     :				Any									=Group(__lowercase					,      __lowercase      ).arrange(__lowercase					,      buff=0.5					,      aligned_edge=__lowercase      )
							cpu.move_to([-2.5, -0.5, 0]      )
							self.add(__lowercase      )

							SCREAMING_SNAKE_CASE__     :				Optional[Any]									=[mem.copy() for i in range(4      )]
							SCREAMING_SNAKE_CASE__     :				Dict									=VGroup(*__lowercase      ).arrange(__lowercase					,      buff=0      )
							SCREAMING_SNAKE_CASE__     :				Optional[Any]									=Text('''GPU'''					,      font_size=24      )
							SCREAMING_SNAKE_CASE__     :				Optional[Any]									=Group(__lowercase					,      __lowercase      ).arrange(__lowercase					,      buff=0.5					,      aligned_edge=__lowercase      )
							gpu.move_to([-1, -1, 0]      )
							self.add(__lowercase      )

							SCREAMING_SNAKE_CASE__     :				Any									=[mem.copy() for i in range(6      )]
							SCREAMING_SNAKE_CASE__     :				Dict									=VGroup(*__lowercase      ).arrange(__lowercase					,      buff=0      )

							SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=Text('''Model'''					,      font_size=24      )
							SCREAMING_SNAKE_CASE__     :				Optional[Any]									=Group(__lowercase					,      __lowercase      ).arrange(__lowercase					,      buff=0.5					,      aligned_edge=__lowercase      )
							model.move_to([3, -1.0, 0]      )
							self.add(__lowercase      )

							SCREAMING_SNAKE_CASE__     :				Tuple									=[]
							SCREAMING_SNAKE_CASE__     :				Tuple									=[]

							for i, rect in enumerate(__lowercase      ):
													SCREAMING_SNAKE_CASE__     :				List[Any]									=fill.copy().set_fill(__lowercase					,      opacity=0.8      )
													target.move_to(__lowercase      )
													model_arr.append(__lowercase      )

													SCREAMING_SNAKE_CASE__     :				str									=Rectangle(height=0.46					,      width=0.46      ).set_stroke(width=0.0      ).set_fill(__lowercase					,      opacity=0.8      )
													cpu_target.move_to(cpu_left_col_base[i]      )
													model_cpu_arr.append(__lowercase      )

							self.add(*__lowercase					,      *__lowercase      )

							SCREAMING_SNAKE_CASE__     :				int									=[meta_mem.copy() for i in range(6      )]
							SCREAMING_SNAKE_CASE__     :				List[Any]									=[meta_mem.copy() for i in range(6      )]
							SCREAMING_SNAKE_CASE__     :				Optional[Any]									=VGroup(*__lowercase      ).arrange(__lowercase					,      buff=0      )
							SCREAMING_SNAKE_CASE__     :				Optional[int]									=VGroup(*__lowercase      ).arrange(__lowercase					,      buff=0      )
							SCREAMING_SNAKE_CASE__     :				List[str]									=VGroup(__lowercase					,      __lowercase      ).arrange(__lowercase					,      buff=0      )
							SCREAMING_SNAKE_CASE__     :				int									=Text('''Disk'''					,      font_size=24      )
							SCREAMING_SNAKE_CASE__     :				Any									=Group(__lowercase					,      __lowercase      ).arrange(__lowercase					,      buff=0.5					,      aligned_edge=__lowercase      )
							disk.move_to([-4, -1.25, 0]      )
							self.add(__lowercase					,      __lowercase      )

							SCREAMING_SNAKE_CASE__     :				Any									=Square(side_length=2.2      )
							key.move_to([-5, 2, 0]      )

							SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=MarkupText(
							    F"<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"					,      font_size=18					,      )

							key_text.move_to([-5, 2.4, 0]      )

							self.add(__lowercase					,      __lowercase      )

							SCREAMING_SNAKE_CASE__     :				Optional[Any]									=MarkupText(
							    F"<span fgcolor='{BLUE}'>●</span> Checkpoint"					,      font_size=18					,      )

							blue_text.next_to(__lowercase					,      DOWN * 2.4					,      aligned_edge=key_text.get_left()      )
							self.add(__lowercase      )

							SCREAMING_SNAKE_CASE__     :				str									=MarkupText(
							    F"Now watch as an input is passed through the model\nand how the memory is utilized and handled."					,      font_size=24					,      )
							step_a.move_to([2, 2, 0]      )

							self.play(Write(__lowercase      )      )

							SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=Square(0.3      )
							input.set_fill(__lowercase					,      opacity=1.0      )
							input.set_stroke(width=0.0      )
							input.next_to(model_base[0]					,      __lowercase					,      buff=0.5      )

							self.play(Write(__lowercase      )      )

							input.generate_target()
							input.target.next_to(model_arr[0]					,      direction=__lowercase					,      buff=0.02      )
							self.play(MoveToTarget(__lowercase      )      )

							self.play(FadeOut(__lowercase      )      )

							SCREAMING_SNAKE_CASE__     :				Optional[Any]									=Arrow(start=__lowercase					,      end=__lowercase					,      color=__lowercase					,      buff=0.5      )
							a.next_to(model_arr[0].get_left()					,      __lowercase					,      buff=0.2      )

							model_cpu_arr[0].generate_target()
							model_cpu_arr[0].target.move_to(gpu_rect[0]      )

							SCREAMING_SNAKE_CASE__     :				Any									=MarkupText(
							    F"As the input reaches a layer, the hook triggers\nand weights are moved from the CPU\nto the GPU and back."					,      font_size=24					,      )
							step_a.move_to([2, 2, 0]      )

							self.play(Write(__lowercase					,      run_time=3      )      )

							SCREAMING_SNAKE_CASE__     :				Tuple									={'''run_time''': 1, '''fade_in''': True, '''fade_out''': True, '''buff''': 0.02}

							self.play(
							    Write(__lowercase      )					,      Circumscribe(model_arr[0]					,      color=__lowercase					,      **__lowercase      )					,      Circumscribe(model_cpu_arr[0]					,      color=__lowercase					,      **__lowercase      )					,      Circumscribe(gpu_rect[0]					,      color=__lowercase					,      **__lowercase      )					,      )
							self.play(MoveToTarget(model_cpu_arr[0]      )      )

							SCREAMING_SNAKE_CASE__     :				Optional[int]									=a.copy()
							for i in range(6      ):
													a_c.next_to(model_arr[i].get_right() + 0.02					,      __lowercase					,      buff=0.2      )

													input.generate_target()
													input.target.move_to(model_arr[i].get_right() + 0.02      )

													SCREAMING_SNAKE_CASE__     :				Optional[Any]									=AnimationGroup(
													    FadeOut(__lowercase					,      run_time=0.5      )					,      MoveToTarget(__lowercase					,      run_time=0.5      )					,      FadeIn(__lowercase					,      run_time=0.5      )					,      lag_ratio=0.2      )

													self.play(__lowercase      )

													model_cpu_arr[i].generate_target()
													model_cpu_arr[i].target.move_to(cpu_left_col_base[i]      )

													if i < 5:
																			model_cpu_arr[i + 1].generate_target()
																			model_cpu_arr[i + 1].target.move_to(gpu_rect[0]      )
																			if i >= 1:
																									SCREAMING_SNAKE_CASE__     :				Optional[int]									=0.7

																			self.play(
																			    Circumscribe(model_arr[i]					,      **__lowercase      )					,      Circumscribe(cpu_left_col_base[i]					,      **__lowercase      )					,      Circumscribe(cpu_left_col_base[i + 1]					,      color=__lowercase					,      **__lowercase      )					,      Circumscribe(gpu_rect[0]					,      color=__lowercase					,      **__lowercase      )					,      Circumscribe(model_arr[i + 1]					,      color=__lowercase					,      **__lowercase      )					,      )
																			if i < 1:
																									self.play(
																									    MoveToTarget(model_cpu_arr[i]      )					,      MoveToTarget(model_cpu_arr[i + 1]      )					,      )
																			else:
																									self.play(
																									    MoveToTarget(model_cpu_arr[i]					,      run_time=0.7      )					,      MoveToTarget(model_cpu_arr[i + 1]					,      run_time=0.7      )					,      )
													else:
																			model_cpu_arr[i].generate_target()
																			model_cpu_arr[i].target.move_to(cpu_left_col_base[-1]      )
																			input.generate_target()
																			input.target.next_to(model_arr[-1].get_right()					,      RIGHT + 0.02					,      buff=0.2      )

																			self.play(
																			    Circumscribe(model_arr[-1]					,      color=__lowercase					,      **__lowercase      )					,      Circumscribe(cpu_left_col_base[-1]					,      color=__lowercase					,      **__lowercase      )					,      Circumscribe(gpu_rect[0]					,      color=__lowercase					,      **__lowercase      )					,      )

																			self.play(MoveToTarget(model_cpu_arr[i]      )      )

													SCREAMING_SNAKE_CASE__     :				List[Any]									=a_c
													SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=a_c.copy()

							input.generate_target()
							input.target.next_to(model_base[-1]					,      RIGHT + 0.02					,      buff=0.5      )
							self.play(
							    FadeOut(__lowercase      )					,      FadeOut(__lowercase					,      run_time=0.5      )					,      )

							SCREAMING_SNAKE_CASE__     :				str									=MarkupText(F"Inference on a model too large for GPU memory\nis successfully completed."					,      font_size=24      )
							step_a.move_to([2, 2, 0]      )

							self.play(Write(__lowercase					,      run_time=3      )					,      MoveToTarget(__lowercase      )      )

							self.wait()







'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_					= logging.get_logger(__name__)

a_					= {
    'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json',
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}






class      __SCREAMING_SNAKE_CASE      (			lowerCamelCase				):
 snake_case_  							=							"""gpt_neox"""




 def __init__(      self	:							List[Any]					,      __lowercase	:							Union[str, Any]=5_04_32					,      __lowercase	:							int=61_44					,      __lowercase	:							Tuple=44					,      __lowercase	:							List[str]=64					,      __lowercase	:							str=2_45_76					,      __lowercase	:							Dict="gelu"					,      __lowercase	:							Tuple=0.25					,      __lowercase	:							Tuple=1_00_00					,      __lowercase	:							Tuple=0.0					,      __lowercase	:							str=0.0					,      __lowercase	:							List[Any]=0.1					,      __lowercase	:							Dict=20_48					,      __lowercase	:							Any=0.02					,      __lowercase	:							Dict=1e-5					,      __lowercase	:							List[Any]=True					,      __lowercase	:							str=0					,      __lowercase	:							Optional[Any]=2					,      __lowercase	:							Tuple=False					,      __lowercase	:							List[Any]=True					,      __lowercase	:							Optional[Any]=None					,      **__lowercase	:							Any					,      )	->						Dict:
       super().__init__(bos_token_id=__lowercase					,      eos_token_id=__lowercase					,      **__lowercase      )
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=vocab_size
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=max_position_embeddings
       SCREAMING_SNAKE_CASE__     :				Any									=hidden_size
       SCREAMING_SNAKE_CASE__     :				str									=num_hidden_layers
       SCREAMING_SNAKE_CASE__     :				Any									=num_attention_heads
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=intermediate_size
       SCREAMING_SNAKE_CASE__     :				Dict									=hidden_act
       SCREAMING_SNAKE_CASE__     :				str									=rotary_pct
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=rotary_emb_base
       SCREAMING_SNAKE_CASE__     :				List[Any]									=attention_dropout
       SCREAMING_SNAKE_CASE__     :				List[Any]									=hidden_dropout
       SCREAMING_SNAKE_CASE__     :				str									=classifier_dropout
       SCREAMING_SNAKE_CASE__     :				Any									=initializer_range
       SCREAMING_SNAKE_CASE__     :				Dict									=layer_norm_eps
       SCREAMING_SNAKE_CASE__     :				Any									=use_cache
       SCREAMING_SNAKE_CASE__     :				Tuple									=tie_word_embeddings
       SCREAMING_SNAKE_CASE__     :				Tuple									=use_parallel_residual
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=rope_scaling
       self._rope_scaling_validation()

       if self.hidden_size % self.num_attention_heads != 0:
             raise ValueError(
                 '''The hidden size is not divisble by the number of attention heads! Make sure to update them!'''      )




 def      __magic_name__	(      self	:							Optional[Any]      )	->						Optional[Any]:
       if self.rope_scaling is None:
             return

       if not isinstance(self.rope_scaling					,      __lowercase      ) or len(self.rope_scaling      ) != 2:
             raise ValueError(
                 '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
                 F"got {self.rope_scaling}"      )
       SCREAMING_SNAKE_CASE__     :				int									=self.rope_scaling.get('''type'''					,      __lowercase      )
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=self.rope_scaling.get('''factor'''					,      __lowercase      )
       if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
             raise ValueError(
                 F"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"      )
       if rope_scaling_factor is None or not isinstance(__lowercase					,      __lowercase      ) or rope_scaling_factor <= 1.0:
             raise ValueError(F"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"      )