


'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


a					  =     logging.get_logger(__name__)

a					  =     {'vocab_file': 'vocab.txt'}

a					  =     {
    'vocab_file': {
        'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt',
        'YituTech/conv-bert-medium-small': (
            'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'
        ),
        'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt',
    }
}

a					  =     {
    'YituTech/conv-bert-base': 5_1_2,
    'YituTech/conv-bert-medium-small': 5_1_2,
    'YituTech/conv-bert-small': 5_1_2,
}


a					  =     {
    'YituTech/conv-bert-base': {'do_lower_case': True},
    'YituTech/conv-bert-medium-small': {'do_lower_case': True},
    'YituTech/conv-bert-small': {'do_lower_case': True},
}







class        UpperCamelCase__   (	__magic_name__   ):
					__SCREAMING_SNAKE_CASE       :  Any						=   VOCAB_FILES_NAMES
					__SCREAMING_SNAKE_CASE       :  Optional[Any]						=   PRETRAINED_VOCAB_FILES_MAP
					__SCREAMING_SNAKE_CASE       :  Dict						=   PRETRAINED_INIT_CONFIGURATION
					__SCREAMING_SNAKE_CASE       :  List[str]						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__SCREAMING_SNAKE_CASE       :  int						=   ConvBertTokenizer

					def __init__(     self       :   Union[str, Any]							,					UpperCamelCase__       :   List[str]=None							,					UpperCamelCase__       :   Any=None							,					UpperCamelCase__       :   Dict=True							,					UpperCamelCase__       :   List[Any]="[UNK]"							,					UpperCamelCase__       :   Dict="[SEP]"							,					UpperCamelCase__       :   Union[str, Any]="[PAD]"							,					UpperCamelCase__       :   Union[str, Any]="[CLS]"							,					UpperCamelCase__       :   Dict="[MASK]"							,					UpperCamelCase__       :   Dict=True							,					UpperCamelCase__       :   Tuple=None							,					**UpperCamelCase__       :   Dict							,					):






										'''simple docstring'''






										super().__init__(
										    UpperCamelCase__							,					tokenizer_file=UpperCamelCase__							,					do_lower_case=UpperCamelCase__							,					unk_token=UpperCamelCase__							,					sep_token=UpperCamelCase__							,					pad_token=UpperCamelCase__							,					cls_token=UpperCamelCase__							,					mask_token=UpperCamelCase__							,					tokenize_chinese_chars=UpperCamelCase__							,					strip_accents=UpperCamelCase__							,					**UpperCamelCase__							,					)

										lowercase_  							=			json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
										if (
										    normalizer_state.get("""lowercase"""							,					UpperCamelCase__       ) != do_lower_case
										    or normalizer_state.get("""strip_accents"""							,					UpperCamelCase__       ) != strip_accents
										    or normalizer_state.get("""handle_chinese_chars"""							,					UpperCamelCase__       ) != tokenize_chinese_chars
										):
															lowercase_  							=			getattr(UpperCamelCase__							,					normalizer_state.pop("""type"""       )       )
															lowercase_  							=			do_lower_case
															lowercase_  							=			strip_accents
															lowercase_  							=			tokenize_chinese_chars
															lowercase_  							=			normalizer_class(**UpperCamelCase__       )

										lowercase_  							=			do_lower_case

					def   UpperCAmelCase__       (     self       :   Union[str, Any]							,					UpperCamelCase__       :   Tuple							,					UpperCamelCase__       :   str=None       ):






										'''simple docstring'''






										lowercase_  							=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

										return output

					def   UpperCAmelCase__       (     self       :   int							,					UpperCamelCase__       :   List[int]							,					UpperCamelCase__       :   Optional[List[int]] = None       ):






										'''simple docstring'''






										lowercase_  							=			[self.sep_token_id]
										lowercase_  							=			[self.cls_token_id]
										if token_ids_a is None:
															return len(cls + token_ids_a + sep       ) * [0]
										return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







					def   UpperCAmelCase__       (     self       :   Optional[Any]							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   Optional[str] = None       ):






										'''simple docstring'''






										lowercase_  							=			self._tokenizer.model.save(UpperCamelCase__							,					name=UpperCamelCase__       )
										return tuple(UpperCamelCase__       )






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_torch_available,
)


a					  =     {
    'configuration_trocr': ['TROCR_PRETRAINED_CONFIG_ARCHIVE_MAP', 'TrOCRConfig'],
    'processing_trocr': ['TrOCRProcessor'],
}


try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							a					  =     [
							    'TROCR_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'TrOCRForCausalLM',
							    'TrOCRPreTrainedModel',
							]


if TYPE_CHECKING:
							from .configuration_trocr import TROCR_PRETRAINED_CONFIG_ARCHIVE_MAP, TrOCRConfig
							from .processing_trocr import TrOCRProcessor

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_trocr import TROCR_PRETRAINED_MODEL_ARCHIVE_LIST, TrOCRForCausalLM, TrOCRPreTrainedModel

else:
							import sys

							a					  =     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


