import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_lowercase							:      Tuple      							={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_lowercase							:      List[str]      							={
    '''vocab_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-generator''': '''https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt''',
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-generator''': (
            '''https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'''
        ),
    },
}

_lowercase							:      int      							={
    '''google/electra-small-generator''': 5_1_2,
    '''google/electra-base-generator''': 5_1_2,
    '''google/electra-large-generator''': 5_1_2,
    '''google/electra-small-discriminator''': 5_1_2,
    '''google/electra-base-discriminator''': 5_1_2,
    '''google/electra-large-discriminator''': 5_1_2,
}

_lowercase							:      Tuple      							={
    '''google/electra-small-generator''': {'''do_lower_case''': True},
    '''google/electra-base-generator''': {'''do_lower_case''': True},
    '''google/electra-large-generator''': {'''do_lower_case''': True},
    '''google/electra-small-discriminator''': {'''do_lower_case''': True},
    '''google/electra-base-discriminator''': {'''do_lower_case''': True},
    '''google/electra-large-discriminator''': {'''do_lower_case''': True},
}



class  SCREAMING_SNAKE_CASE_ (					lowerCAmelCase_			):
  '''simple docstring'''


  lowercase		:     Tuple            =				VOCAB_FILES_NAMES
  lowercase		:     List[Any]            =				PRETRAINED_VOCAB_FILES_MAP
  lowercase		:     Dict            =				PRETRAINED_INIT_CONFIGURATION
  lowercase		:     Union[str, Any]            =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase		:     List[Any]            =				ElectraTokenizer







  def __init__(				self    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]=None      ,	SCREAMING_SNAKE_CASE__    :					Any=None      ,	SCREAMING_SNAKE_CASE__    :					List[str]=True      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]="[UNK]"      ,	SCREAMING_SNAKE_CASE__    :					str="[SEP]"      ,	SCREAMING_SNAKE_CASE__    :					int="[PAD]"      ,	SCREAMING_SNAKE_CASE__    :					Dict="[CLS]"      ,	SCREAMING_SNAKE_CASE__    :					str="[MASK]"      ,	SCREAMING_SNAKE_CASE__    :					int=True      ,	SCREAMING_SNAKE_CASE__    :					Any=None      ,	**SCREAMING_SNAKE_CASE__    :					List[str]      ,	)    ->				Tuple:
         super().__init__(
             SCREAMING_SNAKE_CASE__      ,	tokenizer_file=SCREAMING_SNAKE_CASE__      ,	do_lower_case=SCREAMING_SNAKE_CASE__      ,	unk_token=SCREAMING_SNAKE_CASE__      ,	sep_token=SCREAMING_SNAKE_CASE__      ,	pad_token=SCREAMING_SNAKE_CASE__      ,	cls_token=SCREAMING_SNAKE_CASE__      ,	mask_token=SCREAMING_SNAKE_CASE__      ,	tokenize_chinese_chars=SCREAMING_SNAKE_CASE__      ,	strip_accents=SCREAMING_SNAKE_CASE__      ,	**SCREAMING_SNAKE_CASE__      ,	)

         A						:     Optional[Any]         =json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
         if (
             normalizer_state.get('lowercase'      ,	SCREAMING_SNAKE_CASE__     ) != do_lower_case
             or normalizer_state.get('strip_accents'      ,	SCREAMING_SNAKE_CASE__     ) != strip_accents
             or normalizer_state.get('handle_chinese_chars'      ,	SCREAMING_SNAKE_CASE__     ) != tokenize_chinese_chars
         ):
                A						:     Union[str, Any]         =getattr(SCREAMING_SNAKE_CASE__      ,	normalizer_state.pop('type'     )     )
                A						:     Any         =do_lower_case
                A						:     Any         =strip_accents
                A						:     int         =tokenize_chinese_chars
                A						:     Tuple         =normalizer_class(**SCREAMING_SNAKE_CASE__     )

         A						:     Tuple         =do_lower_case







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					List[str]      ,	SCREAMING_SNAKE_CASE__    :					Any=None     )    ->				Tuple:
         A						:     int         =[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

         return output







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					List[int]      ,	SCREAMING_SNAKE_CASE__    :					Optional[List[int]] = None     )    ->				List[int]:
         A						:     List[str]         =[self.sep_token_id]
         A						:     int         =[self.cls_token_id]
         if token_ids_a is None:
                return len(cls + token_ids_a + sep     ) * [0]
         return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					Optional[str] = None     )    ->				Tuple[str]:
         A						:     Dict         =self._tokenizer.model.save(SCREAMING_SNAKE_CASE__      ,	name=SCREAMING_SNAKE_CASE__     )
         return tuple(SCREAMING_SNAKE_CASE__     )







_lowercase							:      Dict      							='''0.21.0'''

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
    from .utils import rich






