



from manim import *



class 						UpperCAmelCase_					( a):







       def      snake_case__   (     self):

              '''simple docstring'''

              _lowerCAmelCase  :							Any				     =					Rectangle(height=0.5,    width=0.5)
              _lowerCAmelCase  :							Dict				     =					Rectangle(height=0.46,    width=0.46).set_stroke(width=0)

              _lowerCAmelCase  :							Optional[Any]				     =					[mem.copy() for i in range(6)]
              _lowerCAmelCase  :							Any				     =					[mem.copy() for i in range(6)]
              _lowerCAmelCase  :							Any				     =					VGroup(*_UpperCAmelCase).arrange(_UpperCAmelCase,    buff=0)
              _lowerCAmelCase  :							List[str]				     =					VGroup(*_UpperCAmelCase).arrange(_UpperCAmelCase,    buff=0)
              _lowerCAmelCase  :							Dict				     =					VGroup(_UpperCAmelCase,    _UpperCAmelCase).arrange(_UpperCAmelCase,    buff=0)
              _lowerCAmelCase  :							Tuple				     =					Text("CPU",    font_size=24)
              _lowerCAmelCase  :							str				     =					Group(_UpperCAmelCase,    _UpperCAmelCase).arrange(_UpperCAmelCase,    buff=0.5,    aligned_edge=_UpperCAmelCase)
              cpu.move_to([-2.5, -0.5, 0])
              self.add(_UpperCAmelCase)

              _lowerCAmelCase  :							int				     =					[mem.copy() for i in range(4)]
              _lowerCAmelCase  :							Tuple				     =					VGroup(*_UpperCAmelCase).arrange(_UpperCAmelCase,    buff=0)
              _lowerCAmelCase  :							int				     =					Text("GPU",    font_size=24)
              _lowerCAmelCase  :							Tuple				     =					Group(_UpperCAmelCase,    _UpperCAmelCase).arrange(_UpperCAmelCase,    buff=0.5,    aligned_edge=_UpperCAmelCase)
              gpu.move_to([-1, -1, 0])
              self.add(_UpperCAmelCase)

              _lowerCAmelCase  :							List[str]				     =					[mem.copy() for i in range(6)]
              _lowerCAmelCase  :							Union[str, Any]				     =					VGroup(*_UpperCAmelCase).arrange(_UpperCAmelCase,    buff=0)

              _lowerCAmelCase  :							Tuple				     =					Text("Model",    font_size=24)
              _lowerCAmelCase  :							Tuple				     =					Group(_UpperCAmelCase,    _UpperCAmelCase).arrange(_UpperCAmelCase,    buff=0.5,    aligned_edge=_UpperCAmelCase)
              model.move_to([3, -1.0, 0])
              self.add(_UpperCAmelCase)

              _lowerCAmelCase  :							Tuple				     =					[]
              for i, rect in enumerate(_UpperCAmelCase):
                     rect.set_stroke(_UpperCAmelCase)
                     # target = fill.copy().set_fill(YELLOW, opacity=0.7)
                     # target.move_to(rect)
                     # self.add(target)

                     _lowerCAmelCase  :							Union[str, Any]				     =					Rectangle(height=0.46 / 4,    width=0.46 / 3).set_stroke(width=0.0).set_fill(_UpperCAmelCase,    opacity=0.7)

                     if i == 0:
                            cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT),    buff=0.02,    direction=_UpperCAmelCase)
                            cpu_target.set_x(cpu_target.get_x() + 0.1)
                     elif i == 3:
                            cpu_target.next_to(cpu_targs[0],    direction=_UpperCAmelCase,    buff=0.0)
                     else:
                            cpu_target.next_to(cpu_targs[i - 1],    direction=_UpperCAmelCase,    buff=0.0)
                     self.add(_UpperCAmelCase)
                     cpu_targs.append(_UpperCAmelCase)

              _lowerCAmelCase  :							Union[str, Any]				     =					[mem.copy() for i in range(6)]
              _lowerCAmelCase  :							str				     =					VGroup(*_UpperCAmelCase).arrange(_UpperCAmelCase,    buff=0)

              _lowerCAmelCase  :							Dict				     =					Text("Loaded Checkpoint",    font_size=24)
              _lowerCAmelCase  :							Optional[int]				     =					Group(_UpperCAmelCase,    _UpperCAmelCase).arrange(_UpperCAmelCase,    aligned_edge=_UpperCAmelCase,    buff=0.4)
              checkpoint.move_to([3, 0.5, 0])

              _lowerCAmelCase  :							Tuple				     =					Square(side_length=2.2)
              key.move_to([-5, 2, 0])

              _lowerCAmelCase  :							Any				     =					MarkupText(
                  f"<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model",    font_size=18,    )

              key_text.move_to([-5, 2.4, 0])

              self.add(_UpperCAmelCase,    _UpperCAmelCase)

              _lowerCAmelCase  :							Tuple				     =					MarkupText(
                  f"<span fgcolor='{BLUE}'>●</span> Checkpoint",    font_size=18,    )

              blue_text.next_to(_UpperCAmelCase,    DOWN * 2.4,    aligned_edge=key_text.get_left())

              _lowerCAmelCase  :							Any				     =					MarkupText(
                  f"Next, a <i><span fgcolor=\"{BLUE}\">second</span></i> model is loaded into memory,\nwith the weights of a <span fgcolor=\"{BLUE}\">single shard</span>.",    font_size=24,    )
              step_a.move_to([2, 2, 0])
              self.play(Write(_UpperCAmelCase),    Write(_UpperCAmelCase))

              self.play(Write(_UpperCAmelCase,    run_time=1),    Create(_UpperCAmelCase,    run_time=1))

              _lowerCAmelCase  :							Any				     =					[]
              _lowerCAmelCase  :							Union[str, Any]				     =					[]
              for i, rect in enumerate(_UpperCAmelCase):
                     _lowerCAmelCase  :							int				     =					fill.copy().set_fill(_UpperCAmelCase,    opacity=0.7)
                     target.move_to(_UpperCAmelCase)
                     first_animations.append(GrowFromCenter(_UpperCAmelCase,    run_time=1))

                     _lowerCAmelCase  :							List[str]				     =					target.copy()
                     cpu_target.generate_target()
                     if i < 5:
                            cpu_target.target.move_to(cpu_left_col_base[i + 1])
                     else:
                            cpu_target.target.move_to(cpu_right_col_base[i - 5])
                     second_animations.append(MoveToTarget(_UpperCAmelCase,    run_time=1.5))

              self.play(*_UpperCAmelCase)
              self.play(*_UpperCAmelCase)
              self.wait()





import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case	=						logging.get_logger(__name__)

_snake_case	=						{
    "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/config.json",
    "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/config.json",
}



class 						UpperCAmelCase_					( a):
							lowerCamelCase__         =	'xlnet'
							lowerCamelCase__         =	['mems']
							lowerCamelCase__         =	{
							    'n_token': 'vocab_size',  # Backward compatibility
							    'hidden_size': 'd_model',
							    'num_attention_heads': 'n_head',
							    'num_hidden_layers': 'n_layer',
							}







							def __init__(     self,    __a=3_2000,    __a=1024,    __a=24,    __a=16,    __a=4096,    __a="gelu",    __a=True,    __a="bi",    __a=0.02,    __a=1E-12,    __a=0.1,    __a=512,    __a=None,    __a=True,    __a=False,    __a=False,    __a=-1,    __a=False,    __a="last",    __a=True,    __a="tanh",    __a=0.1,    __a=5,    __a=5,    __a=5,    __a=1,    __a=2,    **__a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							int				     =					vocab_size
														_lowerCAmelCase  :							Optional[int]				     =					d_model
														_lowerCAmelCase  :							Tuple				     =					n_layer
														_lowerCAmelCase  :							List[Any]				     =					n_head
														if d_model % n_head != 0:
																					raise ValueError(f"'d_model % n_head' ({d_model % n_head}) should be equal to 0")
														if "d_head" in kwargs:
																					if kwargs["d_head"] != d_model // n_head:
																												raise ValueError(
																												    f"`d_head` ({kwargs['d_head']}) should be equal to `d_model // n_head` ({d_model // n_head})")
														_lowerCAmelCase  :							Optional[int]				     =					d_model // n_head
														_lowerCAmelCase  :							List[str]				     =					ff_activation
														_lowerCAmelCase  :							Tuple				     =					d_inner
														_lowerCAmelCase  :							List[Any]				     =					untie_r
														_lowerCAmelCase  :							List[str]				     =					attn_type

														_lowerCAmelCase  :							Union[str, Any]				     =					initializer_range
														_lowerCAmelCase  :							Any				     =					layer_norm_eps

														_lowerCAmelCase  :							List[Any]				     =					dropout
														_lowerCAmelCase  :							Optional[int]				     =					mem_len
														_lowerCAmelCase  :							Union[str, Any]				     =					reuse_len
														_lowerCAmelCase  :							List[str]				     =					bi_data
														_lowerCAmelCase  :							List[str]				     =					clamp_len
														_lowerCAmelCase  :							Any				     =					same_length

														_lowerCAmelCase  :							List[str]				     =					summary_type
														_lowerCAmelCase  :							int				     =					summary_use_proj
														_lowerCAmelCase  :							Optional[Any]				     =					summary_activation
														_lowerCAmelCase  :							Tuple				     =					summary_last_dropout
														_lowerCAmelCase  :							Union[str, Any]				     =					start_n_top
														_lowerCAmelCase  :							Optional[int]				     =					end_n_top

														_lowerCAmelCase  :							Tuple				     =					bos_token_id
														_lowerCAmelCase  :							List[Any]				     =					pad_token_id
														_lowerCAmelCase  :							Dict				     =					eos_token_id

														if "use_cache" in kwargs:
																					warnings.warn(
																					    "The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`"
																					    " instead.",    __a,    )
																					_lowerCAmelCase  :							Union[str, Any]				     =					kwargs["use_cache"]

														_lowerCAmelCase  :							Union[str, Any]				     =					use_mems_eval
														_lowerCAmelCase  :							Any				     =					use_mems_train
														super().__init__(pad_token_id=__a,    bos_token_id=__a,    eos_token_id=__a,    **__a)







							@property
							def      snake_case__   (     self):

														'''simple docstring'''

														logger.info(f"The model {self.model_type} is one of the few models that has no sequence length limit.")
														return -1







							@max_position_embeddings.setter
							def      snake_case__   (     self,    __a):

														'''simple docstring'''

														raise NotImplementedError(
														    f"The model {self.model_type} is one of the few models that has no sequence length limit.")
