


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase			:  int     =logging.get_logger(__name__)
_UpperCAmelCase			:  Dict     ={"""vocab_file""": """spiece.model"""}

_UpperCAmelCase			:  Union[str, Any]     ={
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

_UpperCAmelCase			:  Any     ={
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

_UpperCAmelCase			:  str     ="""▁"""







class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  List[Any]					=							VOCAB_FILES_NAMES
						SCREAMING_SNAKE_CASE__      :  str					=							PRETRAINED_VOCAB_FILES_MAP
						SCREAMING_SNAKE_CASE__      :  Optional[int]					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




						def __init__(				self   ,						__lowercase   ,						__lowercase=True   ,						__lowercase=True   ,						__lowercase=False   ,						__lowercase="[CLS]"   ,						__lowercase="[SEP]"   ,						__lowercase="<unk>"   ,						__lowercase="[SEP]"   ,						__lowercase="<pad>"   ,						__lowercase="[CLS]"   ,						__lowercase="[MASK]"   ,						__lowercase = None   ,						**__lowercase   ,						)					->    None:
										# Mask token behave like a normal word, i.e. include the space before it and
										# is included in the raw text, there should be a match in a non-normalized sentence.
										lowerCAmelCase_ :      Tuple    				=				(
										    AddedToken(__lowercase   ,						lstrip=__lowercase   ,						rstrip=__lowercase   ,						normalized=__lowercase							)
										    if isinstance(__lowercase   ,						__lowercase							)
										    else mask_token
										)

										lowerCAmelCase_ :      str    				=				{} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=__lowercase   ,						remove_space=__lowercase   ,						keep_accents=__lowercase   ,						bos_token=__lowercase   ,						eos_token=__lowercase   ,						unk_token=__lowercase   ,						sep_token=__lowercase   ,						pad_token=__lowercase   ,						cls_token=__lowercase   ,						mask_token=__lowercase   ,						sp_model_kwargs=self.sp_model_kwargs   ,						**__lowercase   ,						)

										lowerCAmelCase_ :      Tuple    				=				do_lower_case
										lowerCAmelCase_ :      List[Any]    				=				remove_space
										lowerCAmelCase_ :      Optional[int]    				=				keep_accents
										lowerCAmelCase_ :      Any    				=				vocab_file

										lowerCAmelCase_ :      Any    				=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
										self.sp_model.Load(__lowercase							)




						@property
						def 						lowercase_ (				self							)					->    List[str]:
										return len(self.sp_model							)




						def 						lowercase_ (				self							)					->    int:
										lowerCAmelCase_ :      List[Any]    				=				{self.convert_ids_to_tokens(__lowercase							): i for i in range(self.vocab_size							)}
										vocab.update(self.added_tokens_encoder							)
										return vocab




						def __getstate__(				self							)					->    Any:
										lowerCAmelCase_ :      Any    				=				self.__dict__.copy()
										lowerCAmelCase_ :      str    				=				None
										return state




						def __setstate__(				self   ,						__lowercase							)					->    Optional[int]:
										lowerCAmelCase_ :      List[Any]    				=				d

										# for backward compatibility
										if not hasattr(self   ,						'''sp_model_kwargs'''							):
														lowerCAmelCase_ :      int    				=				{}

										lowerCAmelCase_ :      int    				=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
										self.sp_model.Load(self.vocab_file							)




						def 						lowercase_ (				self   ,						__lowercase							)					->    int:
										if self.remove_space:
														lowerCAmelCase_ :      Union[str, Any]    				=				''' '''.join(inputs.strip().split()							)
										else:
														lowerCAmelCase_ :      Optional[Any]    				=				inputs
										lowerCAmelCase_ :      int    				=				outputs.replace('''``'''   ,						'''"'''							).replace('''\'\''''   ,						'''"'''							)

										if not self.keep_accents:
														lowerCAmelCase_ :      int    				=				unicodedata.normalize('''NFKD'''   ,						__lowercase							)
														lowerCAmelCase_ :      str    				=				''''''.join([c for c in outputs if not unicodedata.combining(__lowercase							)]							)
										if self.do_lower_case:
														lowerCAmelCase_ :      int    				=				outputs.lower()

										return outputs




						def 						lowercase_ (				self   ,						__lowercase							)					->    List[str]:
										lowerCAmelCase_ :      Optional[Any]    				=				self.preprocess_text(__lowercase							)
										lowerCAmelCase_ :      Optional[int]    				=				self.sp_model.encode(__lowercase   ,						out_type=__lowercase							)
										lowerCAmelCase_ :      List[str]    				=				[]
										for piece in pieces:
														if len(__lowercase							) > 1 and piece[-1] == str(''','''							) and piece[-2].isdigit():
																		lowerCAmelCase_ :      Any    				=				self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowercase   ,						''''''							)							)
																		if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																						if len(cur_pieces[0]							) == 1:
																										lowerCAmelCase_ :      List[str]    				=				cur_pieces[1:]
																						else:
																										lowerCAmelCase_ :      str    				=				cur_pieces[0][1:]
																		cur_pieces.append(piece[-1]							)
																		new_pieces.extend(__lowercase							)
														else:
																		new_pieces.append(__lowercase							)

										return new_pieces




						def 						lowercase_ (				self   ,						__lowercase							)					->    Optional[Any]:
										return self.sp_model.PieceToId(__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase							)					->    Optional[int]:
										return self.sp_model.IdToPiece(__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase							)					->    Dict:
										lowerCAmelCase_ :      Union[str, Any]    				=				[]
										lowerCAmelCase_ :      Any    				=				''''''
										lowerCAmelCase_ :      Tuple    				=				False
										for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																		if not prev_is_special:
																						out_string += " "
																		out_string += self.sp_model.decode(__lowercase							) + token
																		lowerCAmelCase_ :      List[str]    				=				True
																		lowerCAmelCase_ :      List[Any]    				=				[]
														else:
																		current_sub_tokens.append(__lowercase							)
																		lowerCAmelCase_ :      List[str]    				=				False
										out_string += self.sp_model.decode(__lowercase							)
										return out_string.strip()




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    List[int]:
										lowerCAmelCase_ :      str    				=				[self.sep_token_id]
										lowerCAmelCase_ :      Tuple    				=				[self.cls_token_id]
										if token_ids_a is None:
														return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None   ,						__lowercase = False							)					->    List[int]:

										if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=__lowercase   ,						token_ids_a=__lowercase   ,						already_has_special_tokens=__lowercase							)

										if token_ids_a is not None:
														return [1] + ([0] * len(__lowercase							)) + [1] + ([0] * len(__lowercase							)) + [1]
										return [1] + ([0] * len(__lowercase							)) + [1]




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    List[int]:
										lowerCAmelCase_ :      List[Any]    				=				[self.sep_token_id]
										lowerCAmelCase_ :      Optional[int]    				=				[self.cls_token_id]

										if token_ids_a is None:
														return len(cls + token_ids_a + sep							) * [0]
										return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    Tuple[str]:
										if not os.path.isdir(__lowercase							):
														logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
														return
										lowerCAmelCase_ :      Dict    				=				os.path.join(
										    __lowercase   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

										if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowercase							) and os.path.isfile(self.vocab_file							):
														copyfile(self.vocab_file   ,						__lowercase							)
										elif not os.path.isfile(self.vocab_file							):
														with open(__lowercase   ,						'''wb'''							) as fi:
																		lowerCAmelCase_ :      Tuple    				=				self.sp_model.serialized_model_proto()
																		fi.write(__lowercase							)

										return (out_vocab_file,)



from manim import *







class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						def 						lowercase_ (				self							)					->    Tuple:
										lowerCAmelCase_ :      Dict    				=				Rectangle(height=0.5   ,						width=0.5							)
										lowerCAmelCase_ :      Tuple    				=				Rectangle(height=0.25   ,						width=0.25							)
										lowerCAmelCase_ :      Tuple    				=				Rectangle(height=0.46   ,						width=0.46							).set_stroke(width=0							)

										lowerCAmelCase_ :      Optional[int]    				=				[mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      int    				=				[mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      Optional[int]    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      List[str]    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      int    				=				VGroup(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      Tuple    				=				Text('''CPU'''   ,						font_size=2_4							)
										lowerCAmelCase_ :      Union[str, Any]    				=				Group(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0.5   ,						aligned_edge=__lowercase							)
										cpu.move_to([-2.5, -0.5, 0]							)
										self.add(__lowercase							)

										lowerCAmelCase_ :      List[str]    				=				[mem.copy() for i in range(4							)]
										lowerCAmelCase_ :      Any    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      List[Any]    				=				Text('''GPU'''   ,						font_size=2_4							)
										lowerCAmelCase_ :      int    				=				Group(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0.5   ,						aligned_edge=__lowercase							)
										gpu.move_to([-1, -1, 0]							)
										self.add(__lowercase							)

										lowerCAmelCase_ :      str    				=				[mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      Dict    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)

										lowerCAmelCase_ :      Dict    				=				Text('''Model'''   ,						font_size=2_4							)
										lowerCAmelCase_ :      str    				=				Group(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0.5   ,						aligned_edge=__lowercase							)
										model.move_to([3, -1.0, 0]							)
										self.add(__lowercase							)

										lowerCAmelCase_ :      int    				=				[]
										lowerCAmelCase_ :      int    				=				[]
										lowerCAmelCase_ :      Dict    				=				[]

										for i, rect in enumerate(__lowercase							):
														rect.set_stroke(__lowercase							)

														lowerCAmelCase_ :      Any    				=				Rectangle(height=0.46 / 4   ,						width=0.46 / 3							).set_stroke(width=0.0							).set_fill(__lowercase   ,						opacity=0.7							)

														if i == 0:
																		cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT							)   ,						buff=0.02   ,						direction=__lowercase							)
																		cpu_target.set_x(cpu_target.get_x() + 0.1							)
														elif i == 3:
																		cpu_target.next_to(model_cpu_arr[0]   ,						direction=__lowercase   ,						buff=0.0							)
														else:
																		cpu_target.next_to(model_cpu_arr[i - 1]   ,						direction=__lowercase   ,						buff=0.0							)
														self.add(__lowercase							)
														model_cpu_arr.append(__lowercase							)

										self.add(*__lowercase   ,						*__lowercase   ,						*__lowercase							)

										lowerCAmelCase_ :      Union[str, Any]    				=				[mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      List[str]    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)

										lowerCAmelCase_ :      Union[str, Any]    				=				Text('''Loaded Checkpoint'''   ,						font_size=2_4							)
										lowerCAmelCase_ :      int    				=				Group(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0.5   ,						aligned_edge=__lowercase							)
										checkpoint.move_to([3, 0.5, 0]							)

										self.add(__lowercase							)

										lowerCAmelCase_ :      Optional[Any]    				=				[]
										lowerCAmelCase_ :      Dict    				=				[]

										for i, rect in enumerate(__lowercase							):
														lowerCAmelCase_ :      str    				=				fill.copy().set_fill(__lowercase   ,						opacity=0.7							)
														target.move_to(__lowercase							)
														ckpt_arr.append(__lowercase							)

														lowerCAmelCase_ :      Union[str, Any]    				=				target.copy()
														if i < 5:
																		cpu_target.move_to(cpu_left_col_base[i + 1]							)
														else:
																		cpu_target.move_to(cpu_right_col_base[i - 5]							)
														ckpt_cpu_arr.append(__lowercase							)
										self.add(*__lowercase   ,						*__lowercase							)

										lowerCAmelCase_ :      Union[str, Any]    				=				Square(side_length=2.2							)
										key.move_to([-5, 2, 0]							)

										lowerCAmelCase_ :      str    				=				MarkupText(
										    f"""<b>Key:</b>\n\n<span fgcolor='{YELLOW}'>●</span> Empty Model"""   ,						font_size=1_8   ,						)

										key_text.move_to([-5, 2.4, 0]							)

										self.add(__lowercase   ,						__lowercase							)

										lowerCAmelCase_ :      str    				=				MarkupText(
										    f"""<span fgcolor='{BLUE}'>●</span> Checkpoint"""   ,						font_size=1_8   ,						)

										blue_text.next_to(__lowercase   ,						DOWN * 2.4   ,						aligned_edge=key_text.get_left()							)
										self.add(__lowercase							)

										lowerCAmelCase_ :      str    				=				MarkupText(
										    f"""Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device."""   ,						font_size=2_4   ,						)
										step_a.move_to([2, 2, 0]							)

										lowerCAmelCase_ :      List[Any]    				=				[meta_mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      Any    				=				[meta_mem.copy() for i in range(6							)]
										lowerCAmelCase_ :      Any    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      Union[str, Any]    				=				VGroup(*__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      int    				=				VGroup(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0							)
										lowerCAmelCase_ :      List[str]    				=				Text('''Disk'''   ,						font_size=2_4							)
										lowerCAmelCase_ :      Optional[int]    				=				Group(__lowercase   ,						__lowercase							).arrange(__lowercase   ,						buff=0.5   ,						aligned_edge=__lowercase							)
										disk.move_to([-4.0, -1.25, 0]							)
										self.play(Write(__lowercase   ,						run_time=3							)   ,						Write(__lowercase   ,						run_time=1							)   ,						Create(__lowercase   ,						run_time=1							)							)

										lowerCAmelCase_ :      int    				=				[]
										for i, rect in enumerate(__lowercase							):
														lowerCAmelCase_ :      int    				=				rect.copy()
														target.generate_target()
														target.target.move_to(disk_left_col_base[i]							).scale(0.5							)
														animations.append(MoveToTarget(__lowercase   ,						run_time=1.5							)							)
										self.play(*__lowercase							)

										self.play(FadeOut(__lowercase							)							)

										lowerCAmelCase_ :      Union[str, Any]    				=				MarkupText(f"""Then, the checkpoint is removed from memory\nthrough garbage collection."""   ,						font_size=2_4							)
										step_a.move_to([2, 2, 0]							)

										self.play(Write(__lowercase   ,						run_time=3							)							)

										self.play(
										    FadeOut(__lowercase   ,						__lowercase   ,						*__lowercase   ,						*__lowercase							)   ,						)

										self.wait()