





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


lowercase_		:				Union[str, Any]				=						logging.get_logger(__name__)

lowercase_		:				List[Any]				=						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase_		:				Tuple				=						{
    '''vocab_file''': {
        '''squeezebert/squeezebert-uncased''': (
            '''https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt'''
        ),
        '''squeezebert/squeezebert-mnli''': '''https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt''',
        '''squeezebert/squeezebert-mnli-headless''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''squeezebert/squeezebert-uncased''': (
            '''https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json'''
        ),
        '''squeezebert/squeezebert-mnli''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json'''
        ),
        '''squeezebert/squeezebert-mnli-headless''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json'''
        ),
    },
}

lowercase_		:				Optional[Any]				=						{
    '''squeezebert/squeezebert-uncased''': 512,
    '''squeezebert/squeezebert-mnli''': 512,
    '''squeezebert/squeezebert-mnli-headless''': 512,
}


lowercase_		:				int				=						{
    '''squeezebert/squeezebert-uncased''': {'''do_lower_case''': True},
    '''squeezebert/squeezebert-mnli''': {'''do_lower_case''': True},
    '''squeezebert/squeezebert-mnli-headless''': {'''do_lower_case''': True},
}


class 			lowercase				(							a_     ):



						"""simple docstring"""

						_UpperCamelCase  :	Union[str, Any]								=	VOCAB_FILES_NAMES
						_UpperCamelCase  :	Tuple								=	PRETRAINED_VOCAB_FILES_MAP
						_UpperCamelCase  :	int								=	PRETRAINED_INIT_CONFIGURATION
						_UpperCamelCase  :	List[str]								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_UpperCamelCase  :	Any								=	SqueezeBertTokenizer







						def __init__(    self			:		Tuple ,				lowerCamelCase_			:		str=None ,				lowerCamelCase_			:		Optional[Any]=None ,				lowerCamelCase_			:		Dict=True ,				lowerCamelCase_			:		List[str]="[UNK]" ,				lowerCamelCase_			:		List[Any]="[SEP]" ,				lowerCamelCase_			:		Optional[Any]="[PAD]" ,				lowerCamelCase_			:		Optional[int]="[CLS]" ,				lowerCamelCase_			:		str="[MASK]" ,				lowerCamelCase_			:		str=True ,				lowerCamelCase_			:		List[Any]=None ,				**lowerCamelCase_			:		str ,				):





										'''simple docstring'''







										super().__init__(
										    lowerCamelCase_ ,				tokenizer_file=lowerCamelCase_ ,				do_lower_case=lowerCamelCase_ ,				unk_token=lowerCamelCase_ ,				sep_token=lowerCamelCase_ ,				pad_token=lowerCamelCase_ ,				cls_token=lowerCamelCase_ ,				mask_token=lowerCamelCase_ ,				tokenize_chinese_chars=lowerCamelCase_ ,				strip_accents=lowerCamelCase_ ,				**lowerCamelCase_ ,				)

										_snake_case :			Optional[Any]  						=   json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
										if (
										    normalizer_state.get('lowercase' ,				lowerCamelCase_					) != do_lower_case
										    or normalizer_state.get('strip_accents' ,				lowerCamelCase_					) != strip_accents
										    or normalizer_state.get('handle_chinese_chars' ,				lowerCamelCase_					) != tokenize_chinese_chars
										):
														_snake_case :			Dict  						=   getattr(lowerCamelCase_ ,				normalizer_state.pop('type'					)					)
														_snake_case :			Union[str, Any]  						=   do_lower_case
														_snake_case :			str  						=   strip_accents
														_snake_case :			Optional[int]  						=   tokenize_chinese_chars
														_snake_case :			List[Any]  						=   normalizer_class(**lowerCamelCase_					)

										_snake_case :			Union[str, Any]  						=   do_lower_case







						def 		__UpperCAmelCase       (    self			:		Optional[Any] ,				lowerCamelCase_			:		Dict ,				lowerCamelCase_			:		Optional[Any]=None					):





										'''simple docstring'''







										_snake_case :			str  						=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
														output += token_ids_a + [self.sep_token_id]

										return output







						def 		__UpperCAmelCase       (    self			:		List[Any] ,				lowerCamelCase_			:		List[int] ,				lowerCamelCase_			:		Optional[List[int]] = None					):





										'''simple docstring'''







										_snake_case :			Optional[Any]  						=   [self.sep_token_id]
										_snake_case :			List[str]  						=   [self.cls_token_id]
										if token_ids_a is None:
														return len(cls + token_ids_a + sep					) * [0]
										return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]







						def 		__UpperCAmelCase       (    self			:		Optional[Any] ,				lowerCamelCase_			:		str ,				lowerCamelCase_			:		Optional[str] = None					):





										'''simple docstring'''







										_snake_case :			Optional[int]  						=   self._tokenizer.model.save(lowerCamelCase_ ,				name=lowerCamelCase_					)
										return tuple(lowerCamelCase_					)













lowercase_		:				Tuple				=						'''
# Installazione di Transformers
! pip install transformers datasets
# Per installare dalla fonte invece dell\'ultima versione rilasciata, commenta il comando sopra e
# rimuovi la modalità commento al comando seguente.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

lowercase_		:				Optional[int]				=						[{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
lowercase_		:				str				=						{
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}






