
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowerCamelCase_						: int         =  logging.get_logger(__name__)

lowerCamelCase_						: str         =  {"""vocab_file""": """vocab.txt"""}

lowerCamelCase_						: List[Any]         =  {
    """vocab_file""": {
        """YituTech/conv-bert-base""": """https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt""",
        """YituTech/conv-bert-medium-small""": (
            """https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"""
        ),
        """YituTech/conv-bert-small""": """https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt""",
    }
}

lowerCamelCase_						: Tuple         =  {
    """YituTech/conv-bert-base""": 5_12,
    """YituTech/conv-bert-medium-small""": 5_12,
    """YituTech/conv-bert-small""": 5_12,
}


lowerCamelCase_						: List[Any]         =  {
    """YituTech/conv-bert-base""": {"""do_lower_case""": True},
    """YituTech/conv-bert-medium-small""": {"""do_lower_case""": True},
    """YituTech/conv-bert-small""": {"""do_lower_case""": True},
}

class        _UpperCamelCase				(					_A  ):






			'''simple docstring'''

			__UpperCamelCase      :						Any					      = VOCAB_FILES_NAMES
			__UpperCamelCase      :						Any					      = PRETRAINED_VOCAB_FILES_MAP
			__UpperCamelCase      :						List[Any]					      = PRETRAINED_INIT_CONFIGURATION
			__UpperCamelCase      :						Union[str, Any]					      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__UpperCamelCase      :						Optional[Any]					      = ConvBertTokenizer

			def __init__(       self :       Dict						,	snake_case_ :       str=None						,	snake_case_ :       int=None						,	snake_case_ :       List[Any]=True						,	snake_case_ :       List[Any]="[UNK]"						,	snake_case_ :       List[Any]="[SEP]"						,	snake_case_ :       Optional[Any]="[PAD]"						,	snake_case_ :       List[Any]="[CLS]"						,	snake_case_ :       Any="[MASK]"						,	snake_case_ :       Optional[int]=True						,	snake_case_ :       Dict=None						,	**snake_case_ :       Dict						,	):
							super().__init__(
							    snake_case_						,	tokenizer_file=snake_case_						,	do_lower_case=snake_case_						,	unk_token=snake_case_						,	sep_token=snake_case_						,	pad_token=snake_case_						,	cls_token=snake_case_						,	mask_token=snake_case_						,	tokenize_chinese_chars=snake_case_						,	strip_accents=snake_case_						,	**snake_case_						,	)

							UpperCamelCase_:							Dict	=						json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
							if (
							    normalizer_state.get("""lowercase"""						,	snake_case_      ) != do_lower_case
							    or normalizer_state.get("""strip_accents"""						,	snake_case_      ) != strip_accents
							    or normalizer_state.get("""handle_chinese_chars"""						,	snake_case_      ) != tokenize_chinese_chars
							):
											UpperCamelCase_:							int	=						getattr(snake_case_						,	normalizer_state.pop("""type"""      )      )
											UpperCamelCase_:							List[str]	=						do_lower_case
											UpperCamelCase_:							Any	=						strip_accents
											UpperCamelCase_:							Any	=						tokenize_chinese_chars
											UpperCamelCase_:							Tuple	=						normalizer_class(**snake_case_      )

							UpperCamelCase_:							Dict	=						do_lower_case

			def       lowerCAmelCase__	(       self :       Optional[Any]						,	snake_case_ :       List[Any]						,	snake_case_ :       Dict=None      ):
							UpperCamelCase_:							Optional[int]	=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

							return output

			def       lowerCAmelCase__	(       self :       List[str]						,	snake_case_ :       List[int]						,	snake_case_ :       Optional[List[int]] = None      ):
							UpperCamelCase_:							Dict	=						[self.sep_token_id]
							UpperCamelCase_:							Optional[Any]	=						[self.cls_token_id]
							if token_ids_a is None:
											return len(cls + token_ids_a + sep      ) * [0]
							return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]




			def       lowerCAmelCase__	(       self :       Tuple						,	snake_case_ :       str						,	snake_case_ :       Optional[str] = None      ):
							UpperCamelCase_:							Any	=						self._tokenizer.model.save(snake_case_						,	name=snake_case_      )
							return tuple(snake_case_      )








from manim import *

class        _UpperCamelCase				(					_A  ):






			'''simple docstring'''

			def       lowerCAmelCase__	(       self :       int      ):
							UpperCamelCase_:							Dict	=						Rectangle(height=0.5						,	width=0.5      )
							UpperCamelCase_:							Dict	=						Rectangle(height=0.46						,	width=0.46      ).set_stroke(width=0      )

							UpperCamelCase_:							Tuple	=						[mem.copy() for i in range(6      )]
							UpperCamelCase_:							List[str]	=						[mem.copy() for i in range(6      )]
							UpperCamelCase_:							List[str]	=						VGroup(*snake_case_      ).arrange(snake_case_						,	buff=0      )
							UpperCamelCase_:							Tuple	=						VGroup(*snake_case_      ).arrange(snake_case_						,	buff=0      )
							UpperCamelCase_:							Union[str, Any]	=						VGroup(snake_case_						,	snake_case_      ).arrange(snake_case_						,	buff=0      )
							UpperCamelCase_:							Optional[Any]	=						Text("""CPU"""						,	font_size=24      )
							UpperCamelCase_:							int	=						Group(snake_case_						,	snake_case_      ).arrange(snake_case_						,	buff=0.5						,	aligned_edge=snake_case_      )
							cpu.move_to([-2.5, -0.5, 0]      )
							self.add(snake_case_      )

							UpperCamelCase_:							Optional[int]	=						[mem.copy() for i in range(1      )]
							UpperCamelCase_:							Dict	=						VGroup(*snake_case_      ).arrange(snake_case_						,	buff=0      )
							UpperCamelCase_:							Optional[int]	=						Text("""GPU"""						,	font_size=24      )
							UpperCamelCase_:							Optional[int]	=						Group(snake_case_						,	snake_case_      ).arrange(snake_case_						,	buff=0.5						,	aligned_edge=snake_case_      )
							gpu.align_to(snake_case_						,	snake_case_      )
							gpu.set_x(gpu.get_x() - 1      )

							self.add(snake_case_      )

							UpperCamelCase_:							Dict	=						[mem.copy() for i in range(6      )]
							UpperCamelCase_:							List[str]	=						VGroup(*snake_case_      ).arrange(snake_case_						,	buff=0      )

							UpperCamelCase_:							Any	=						Text("""Model"""						,	font_size=24      )
							UpperCamelCase_:							Optional[Any]	=						Group(snake_case_						,	snake_case_      ).arrange(snake_case_						,	buff=0.5						,	aligned_edge=snake_case_      )
							model.move_to([3, -1.0, 0]      )

							self.play(
							    Create(snake_case_						,	run_time=1      )						,	Create(snake_case_						,	run_time=1      )						,	Create(snake_case_						,	run_time=1      )						,	)

							UpperCamelCase_:							List[Any]	=						MarkupText(
							    f'''First, an empty model skeleton is loaded\ninto <span fgcolor=\'{YELLOW}\'>memory</span> without using much RAM.'''						,	font_size=24						,	)

							UpperCamelCase_:							Optional[Any]	=						Square(side_length=2.2      )
							key.move_to([-5, 2, 0]      )

							UpperCamelCase_:							Union[str, Any]	=						MarkupText(
							    f'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''						,	font_size=18						,	)

							key_text.move_to([-5, 2.4, 0]      )

							step_a.move_to([2, 2, 0]      )
							self.play(Write(snake_case_						,	run_time=2.5      )						,	Write(snake_case_      )						,	Write(snake_case_      )      )

							self.add(snake_case_      )

							UpperCamelCase_:							Union[str, Any]	=						[]
							UpperCamelCase_:							Union[str, Any]	=						[]
							UpperCamelCase_:							Tuple	=						[]
							for i, rect in enumerate(snake_case_      ):
											UpperCamelCase_:							Tuple	=						Rectangle(height=0.46						,	width=0.46      ).set_stroke(width=0.0      ).set_fill(snake_case_						,	opacity=0.7      )
											cpu_target.move_to(snake_case_      )
											cpu_target.generate_target()
											UpperCamelCase_:							int	=						0.46 / 4
											UpperCamelCase_:							Optional[int]	=						0.46 / 3

											if i == 0:
															cpu_target.target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT      )						,	buff=0.02						,	direction=snake_case_      )
															cpu_target.target.set_x(cpu_target.target.get_x() + 0.1      )
											elif i == 3:
															cpu_target.target.next_to(cpu_targs[0].target						,	direction=snake_case_						,	buff=0.0      )
											else:
															cpu_target.target.next_to(cpu_targs[i - 1].target						,	direction=snake_case_						,	buff=0.0      )
											cpu_targs.append(snake_case_      )

											first_animations.append(rect.animate(run_time=0.5      ).set_stroke(snake_case_      )      )
											second_animations.append(MoveToTarget(snake_case_						,	run_time=1.5      )      )

							self.play(*snake_case_      )
							self.play(*snake_case_      )

							self.wait()






