


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


__a     =			logging.get_logger(__name__)

__a     =			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__a     =			{
    """vocab_file""": {
        """yjernite/retribert-base-uncased""": (
            """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """yjernite/retribert-base-uncased""": (
            """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

__a     =			{
    """yjernite/retribert-base-uncased""": 512,
}


__a     =			{
    """yjernite/retribert-base-uncased""": {"""do_lower_case""": True},
}






class   __lowercase				(  __snake_case  ):
      UpperCamelCase								=       VOCAB_FILES_NAMES
      UpperCamelCase								=       PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase								=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase								=       PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase								=       RetriBertTokenizer
      UpperCamelCase								=       ['''input_ids''', '''attention_mask''']





      def __init__(		self						:					Optional[int]					,       __lowerCamelCase						:					Tuple=None					,       __lowerCamelCase						:					str=None					,       __lowerCamelCase						:					Optional[Any]=True					,       __lowerCamelCase						:					int="[UNK]"					,       __lowerCamelCase						:					Dict="[SEP]"					,       __lowerCamelCase						:					Union[str, Any]="[PAD]"					,       __lowerCamelCase						:					Optional[Any]="[CLS]"					,       __lowerCamelCase						:					str="[MASK]"					,       __lowerCamelCase						:					Union[str, Any]=True					,       __lowerCamelCase						:					int=None					,       **__lowerCamelCase						:					Any					,       )    ->   Any:






         """simple docstring"""







         super().__init__(
             __lowerCamelCase					,       tokenizer_file=__lowerCamelCase					,       do_lower_case=__lowerCamelCase					,       unk_token=__lowerCamelCase					,       sep_token=__lowerCamelCase					,       pad_token=__lowerCamelCase					,       cls_token=__lowerCamelCase					,       mask_token=__lowerCamelCase					,       tokenize_chinese_chars=__lowerCamelCase					,       strip_accents=__lowerCamelCase					,       **__lowerCamelCase					,       )

         UpperCAmelCase							=     json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
         if (
             normalizer_state.get("""lowercase"""					,       __lowerCamelCase       ) != do_lower_case
             or normalizer_state.get("""strip_accents"""					,       __lowerCamelCase       ) != strip_accents
             or normalizer_state.get("""handle_chinese_chars"""					,       __lowerCamelCase       ) != tokenize_chinese_chars
         ):
            UpperCAmelCase							=     getattr(__lowerCamelCase					,       normalizer_state.pop("""type"""       )       )
            UpperCAmelCase							=     do_lower_case
            UpperCAmelCase							=     strip_accents
            UpperCAmelCase							=     tokenize_chinese_chars
            UpperCAmelCase							=     normalizer_class(**__lowerCamelCase       )

         UpperCAmelCase							=     do_lower_case





      def        _lowercase				(		self						:					int					,       __lowerCamelCase						:					str					,       __lowerCamelCase						:					Dict=None       )    ->   Optional[int]:






         """simple docstring"""







         UpperCAmelCase							=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

         return output





      def        _lowercase				(		self						:					Optional[Any]					,       __lowerCamelCase						:					List[int]					,       __lowerCamelCase						:					Optional[List[int]] = None       )    ->   List[int]:






         """simple docstring"""







         UpperCAmelCase							=     [self.sep_token_id]
         UpperCAmelCase							=     [self.cls_token_id]
         if token_ids_a is None:
            return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





      def        _lowercase				(		self						:					Optional[int]					,       __lowerCamelCase						:					str					,       __lowerCamelCase						:					Optional[str] = None       )    ->   Tuple[str]:






         """simple docstring"""







         UpperCAmelCase							=     self._tokenizer.model.save(__lowerCamelCase					,       name=__lowerCamelCase       )
         return tuple(__lowerCamelCase       )







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__a     =			logging.get_logger(__name__)

__a     =			{
    """facebook/nllb-moe-54B""": """https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json""",
}






class   __lowercase				(  __snake_case  ):
      UpperCamelCase								=       '''nllb-moe'''
      UpperCamelCase								=       ['''past_key_values''']
      UpperCamelCase								=       {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}





      def __init__(		self						:					Optional[int]					,       __lowerCamelCase						:					Optional[Any]=1_2_8_1_1_2					,       __lowerCamelCase						:					Dict=1_0_2_4					,       __lowerCamelCase						:					Optional[int]=1_2					,       __lowerCamelCase						:					Union[str, Any]=4_0_9_6					,       __lowerCamelCase						:					List[str]=1_6					,       __lowerCamelCase						:					List[str]=1_2					,       __lowerCamelCase						:					int=4_0_9_6					,       __lowerCamelCase						:					Tuple=1_6					,       __lowerCamelCase						:					str=0.05					,       __lowerCamelCase						:					List[str]=0.05					,       __lowerCamelCase						:					List[Any]=True					,       __lowerCamelCase						:					Tuple=True					,       __lowerCamelCase						:					str="relu"					,       __lowerCamelCase						:					Dict=1_0_2_4					,       __lowerCamelCase						:					List[str]=0.1					,       __lowerCamelCase						:					Optional[int]=0.1					,       __lowerCamelCase						:					List[Any]=0.0					,       __lowerCamelCase						:					Optional[Any]=0.02					,       __lowerCamelCase						:					Dict=2					,       __lowerCamelCase						:					Union[str, Any]=True					,       __lowerCamelCase						:					Any=False					,       __lowerCamelCase						:					Tuple="float32"					,       __lowerCamelCase						:					Any=False					,       __lowerCamelCase						:					Optional[int]=1_2_8					,       __lowerCamelCase						:					List[str]=6_4					,       __lowerCamelCase						:					List[Any]=4					,       __lowerCamelCase						:					Tuple=4					,       __lowerCamelCase						:					str=0.001					,       __lowerCamelCase						:					Optional[int]=0.001					,       __lowerCamelCase						:					Tuple="all"					,       __lowerCamelCase						:					Any=False					,       __lowerCamelCase						:					Optional[int]=False					,       __lowerCamelCase						:					List[str]=1.0					,       __lowerCamelCase						:					Dict=0.2					,       __lowerCamelCase						:					Union[str, Any]=1					,       __lowerCamelCase						:					int=0					,       __lowerCamelCase						:					Dict=2					,       __lowerCamelCase						:					int=False					,       **__lowerCamelCase						:					str					,       )    ->   int:






         """simple docstring"""







         UpperCAmelCase							=     vocab_size
         UpperCAmelCase							=     max_position_embeddings
         UpperCAmelCase							=     d_model
         UpperCAmelCase							=     encoder_ffn_dim
         UpperCAmelCase							=     encoder_layers
         UpperCAmelCase							=     encoder_attention_heads
         UpperCAmelCase							=     decoder_ffn_dim
         UpperCAmelCase							=     decoder_layers
         UpperCAmelCase							=     decoder_attention_heads
         UpperCAmelCase							=     dropout
         UpperCAmelCase							=     attention_dropout
         UpperCAmelCase							=     activation_dropout
         UpperCAmelCase							=     activation_function
         UpperCAmelCase							=     init_std
         UpperCAmelCase							=     encoder_layerdrop
         UpperCAmelCase							=     decoder_layerdrop
         UpperCAmelCase							=     use_cache
         UpperCAmelCase							=     encoder_layers
         UpperCAmelCase							=     scale_embedding  # scale factor will be sqrt(d_model) if True
         UpperCAmelCase							=     router_z_loss_coef
         UpperCAmelCase							=     router_aux_loss_coef
         UpperCAmelCase							=     decoder_sparse_step
         UpperCAmelCase							=     encoder_sparse_step
         UpperCAmelCase							=     num_experts
         UpperCAmelCase							=     expert_capacity
         UpperCAmelCase							=     router_bias
         if router_dtype not in ["float32", "float16", "bfloat16"]:
            raise ValueError(F"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""       )
         UpperCAmelCase							=     router_dtype

         UpperCAmelCase							=     router_ignore_padding_tokens
         UpperCAmelCase							=     batch_prioritized_routing
         UpperCAmelCase							=     second_expert_policy
         UpperCAmelCase							=     normalize_router_prob_before_dropping
         UpperCAmelCase							=     moe_eval_capacity_token_fraction
         UpperCAmelCase							=     moe_token_dropout
         UpperCAmelCase							=     output_router_logits
         super().__init__(
             pad_token_id=__lowerCamelCase					,       bos_token_id=__lowerCamelCase					,       eos_token_id=__lowerCamelCase					,       is_encoder_decoder=__lowerCamelCase					,       decoder_start_token_id=__lowerCamelCase					,       **__lowerCamelCase					,       )



