






'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase       		=      logging.get_logger(__name__)

__lowerCAmelCase       		=      {
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				"nllb-moe"
			snake_case__  =				["past_key_values"]
			snake_case__  =				{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}






			def __init__( self      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			int=12_8112  ,						__SCREAMING_SNAKE_CASE      :			List[str]=1024  ,						__SCREAMING_SNAKE_CASE      :			Any=12  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=4096  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=16  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]=12  ,						__SCREAMING_SNAKE_CASE      :			Dict=4096  ,						__SCREAMING_SNAKE_CASE      :			Dict=16  ,						__SCREAMING_SNAKE_CASE      :			Dict=0.05  ,						__SCREAMING_SNAKE_CASE      :			int=0.05  ,						__SCREAMING_SNAKE_CASE      :			Dict=True  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]=True  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]="relu"  ,						__SCREAMING_SNAKE_CASE      :			int=1024  ,						__SCREAMING_SNAKE_CASE      :			Union[str, Any]=0.1  ,						__SCREAMING_SNAKE_CASE      :			str=0.1  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]=0.0  ,						__SCREAMING_SNAKE_CASE      :			Dict=0.02  ,						__SCREAMING_SNAKE_CASE      :			int=2  ,						__SCREAMING_SNAKE_CASE      :			Any=True  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=False  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]="float32"  ,						__SCREAMING_SNAKE_CASE      :			Dict=False  ,						__SCREAMING_SNAKE_CASE      :			Union[str, Any]=128  ,						__SCREAMING_SNAKE_CASE      :			str=64  ,						__SCREAMING_SNAKE_CASE      :			Tuple=4  ,						__SCREAMING_SNAKE_CASE      :			str=4  ,						__SCREAMING_SNAKE_CASE      :			List[str]=0.001  ,						__SCREAMING_SNAKE_CASE      :			str=0.001  ,						__SCREAMING_SNAKE_CASE      :			str="all"  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=False  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=False  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]=1.0  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=0.2  ,						__SCREAMING_SNAKE_CASE      :			str=1  ,						__SCREAMING_SNAKE_CASE      :			Any=0  ,						__SCREAMING_SNAKE_CASE      :			int=2  ,						__SCREAMING_SNAKE_CASE      :			Tuple=False  ,						**__SCREAMING_SNAKE_CASE      :			Any  ,						)			->			int:
							a_		:  List[Any]											=      vocab_size
							a_		:  List[str]											=      max_position_embeddings
							a_		:  Any											=      d_model
							a_		:  Optional[Any]											=      encoder_ffn_dim
							a_		:  int											=      encoder_layers
							a_		:  Optional[Any]											=      encoder_attention_heads
							a_		:  int											=      decoder_ffn_dim
							a_		:  Optional[int]											=      decoder_layers
							a_		:  Optional[Any]											=      decoder_attention_heads
							a_		:  Optional[Any]											=      dropout
							a_		:  List[Any]											=      attention_dropout
							a_		:  int											=      activation_dropout
							a_		:  str											=      activation_function
							a_		:  List[Any]											=      init_std
							a_		:  Optional[int]											=      encoder_layerdrop
							a_		:  Optional[int]											=      decoder_layerdrop
							a_		:  Any											=      use_cache
							a_		:  Any											=      encoder_layers
							a_		:  int											=      scale_embedding  # scale factor will be sqrt(d_model) if True
							a_		:  Optional[Any]											=      router_z_loss_coef
							a_		:  Tuple											=      router_aux_loss_coef
							a_		:  List[str]											=      decoder_sparse_step
							a_		:  List[Any]											=      encoder_sparse_step
							a_		:  str											=      num_experts
							a_		:  Union[str, Any]											=      expert_capacity
							a_		:  Optional[int]											=      router_bias
							if router_dtype not in ["float32", "float16", "bfloat16"]:
											raise ValueError(f'`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'					)
							a_		:  List[Any]											=      router_dtype

							a_		:  int											=      router_ignore_padding_tokens
							a_		:  str											=      batch_prioritized_routing
							a_		:  int											=      second_expert_policy
							a_		:  List[Any]											=      normalize_router_prob_before_dropping
							a_		:  List[str]											=      moe_eval_capacity_token_fraction
							a_		:  Any											=      moe_token_dropout
							a_		:  List[Any]											=      output_router_logits
							super().__init__(
							    pad_token_id=__SCREAMING_SNAKE_CASE  ,						bos_token_id=__SCREAMING_SNAKE_CASE  ,						eos_token_id=__SCREAMING_SNAKE_CASE  ,						is_encoder_decoder=__SCREAMING_SNAKE_CASE  ,						decoder_start_token_id=__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE  ,						)












'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase       		=      logging.get_logger(__name__)

__lowerCAmelCase       		=      {'vocab_file': 'sentencepiece.bpe.model'}

__lowerCAmelCase       		=      {
    'vocab_file': {
        'moussaKam/mbarthez': 'https://huggingface.co/moussaKam/mbarthez/resolve/main/sentencepiece.bpe.model',
        'moussaKam/barthez': 'https://huggingface.co/moussaKam/barthez/resolve/main/sentencepiece.bpe.model',
        'moussaKam/barthez-orangesum-title': (
            'https://huggingface.co/moussaKam/barthez-orangesum-title/resolve/main/sentencepiece.bpe.model'
        ),
    },
}

__lowerCAmelCase       		=      {
    'moussaKam/mbarthez': 1_024,
    'moussaKam/barthez': 1_024,
    'moussaKam/barthez-orangesum-title': 1_024,
}

__lowerCAmelCase       		=      '▁'





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				VOCAB_FILES_NAMES
			snake_case__  =				PRETRAINED_VOCAB_FILES_MAP
			snake_case__  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case__  =				["input_ids", "attention_mask"]






			def __init__( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			str  ,						__SCREAMING_SNAKE_CASE      :			Dict="<s>"  ,						__SCREAMING_SNAKE_CASE      :			List[Any]="</s>"  ,						__SCREAMING_SNAKE_CASE      :			List[str]="</s>"  ,						__SCREAMING_SNAKE_CASE      :			List[str]="<s>"  ,						__SCREAMING_SNAKE_CASE      :			Dict="<unk>"  ,						__SCREAMING_SNAKE_CASE      :			int="<pad>"  ,						__SCREAMING_SNAKE_CASE      :			Tuple="<mask>"  ,						__SCREAMING_SNAKE_CASE      :			Optional[Dict[str, Any]] = None  ,						**__SCREAMING_SNAKE_CASE      :			Optional[Any]  ,						)			->			None:
							# Mask token behave like a normal word, i.e. include the space before it
							a_		:  Tuple											=      AddedToken(__SCREAMING_SNAKE_CASE  ,						lstrip=__SCREAMING_SNAKE_CASE  ,						rstrip=__SCREAMING_SNAKE_CASE					) if isinstance(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					) else mask_token

							a_		:  List[str]											=      {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=__SCREAMING_SNAKE_CASE  ,						eos_token=__SCREAMING_SNAKE_CASE  ,						unk_token=__SCREAMING_SNAKE_CASE  ,						sep_token=__SCREAMING_SNAKE_CASE  ,						cls_token=__SCREAMING_SNAKE_CASE  ,						pad_token=__SCREAMING_SNAKE_CASE  ,						mask_token=__SCREAMING_SNAKE_CASE  ,						sp_model_kwargs=self.sp_model_kwargs  ,						**__SCREAMING_SNAKE_CASE  ,						)

							a_		:  Tuple											=      vocab_file
							a_		:  Optional[Any]											=      spm.SentencePieceProcessor(**self.sp_model_kwargs					)
							self.sp_model.Load(str(__SCREAMING_SNAKE_CASE					)					)

							a_		:  Optional[Any]											=      {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

							a_		:  Any											=      len(self.sp_model					) - 1
							a_		:  Optional[Any]											=      {v: k for k, v in self.fairseq_tokens_to_ids.items()}






			def 			SCREAMING_SNAKE_CASE      ( self      :			str  ,						__SCREAMING_SNAKE_CASE      :			List[int]  ,						__SCREAMING_SNAKE_CASE      :			Optional[List[int]] = None					)			->			List[int]:

							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							a_		:  List[str]											=      [self.cls_token_id]
							a_		:  Optional[Any]											=      [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]  ,						__SCREAMING_SNAKE_CASE      :			List[int]  ,						__SCREAMING_SNAKE_CASE      :			Optional[List[int]] = None  ,						__SCREAMING_SNAKE_CASE      :			bool = False					)			->			List[int]:
							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=__SCREAMING_SNAKE_CASE  ,						token_ids_a=__SCREAMING_SNAKE_CASE  ,						already_has_special_tokens=__SCREAMING_SNAKE_CASE					)

							if token_ids_a is None:
											return [1] + ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1]
							return [1] + ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1]






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			List[int]  ,						__SCREAMING_SNAKE_CASE      :			Optional[List[int]] = None					)			->			List[int]:
							a_		:  List[str]											=      [self.sep_token_id]
							a_		:  Any											=      [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep					) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Any					)			->			Union[str, Any]:
							return len(self.sp_model					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			str					)			->			Tuple:
							a_		:  int											=      {self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE					): i for i in range(self.vocab_size					)}
							vocab.update(self.added_tokens_encoder					)
							return vocab






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]  ,						__SCREAMING_SNAKE_CASE      :			str					)			->			List[str]:
							return self.sp_model.encode(__SCREAMING_SNAKE_CASE  ,						out_type=__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			str  ,						__SCREAMING_SNAKE_CASE      :			List[Any]					)			->			Dict:
							if token in self.fairseq_tokens_to_ids:
											return self.fairseq_tokens_to_ids[token]
							a_		:  Optional[int]											=      self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE					)

							return spm_id if spm_id else self.unk_token_id






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]  ,						__SCREAMING_SNAKE_CASE      :			int					)			->			str:
							if index in self.fairseq_ids_to_tokens:
											return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			str  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]					)			->			Optional[int]:
							a_		:  Dict											=      []
							a_		:  List[Any]											=      ''''''
							a_		:  Dict											=      False
							for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
															if not prev_is_special:
																			out_string += " "
															out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE					) + token
															a_		:  Dict											=      True
															a_		:  Optional[Any]											=      []
											else:
															current_sub_tokens.append(__SCREAMING_SNAKE_CASE					)
															a_		:  Tuple											=      False
							out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE					)
							return out_string.strip()






			def __getstate__( self      :			Dict					)			->			int:
							a_		:  Dict											=      self.__dict__.copy()
							a_		:  List[str]											=      None
							return state






			def __setstate__( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]					)			->			Any:
							a_		:  Optional[Any]											=      d

							# for backward compatibility
							if not hasattr(self  ,						'''sp_model_kwargs'''					):
											a_		:  Union[str, Any]											=      {}

							a_		:  int											=      spm.SentencePieceProcessor(**self.sp_model_kwargs					)
							self.sp_model.Load(self.vocab_file					)







			def 			SCREAMING_SNAKE_CASE      ( self      :			str  ,						__SCREAMING_SNAKE_CASE      :			str  ,						__SCREAMING_SNAKE_CASE      :			Optional[str] = None					)			->			Tuple[str]:
							if not os.path.isdir(__SCREAMING_SNAKE_CASE					):
											logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
											return
							a_		:  Union[str, Any]											=      os.path.join(
							    __SCREAMING_SNAKE_CASE  ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

							if os.path.abspath(self.vocab_file					) != os.path.abspath(__SCREAMING_SNAKE_CASE					) and os.path.isfile(self.vocab_file					):
											copyfile(self.vocab_file  ,						__SCREAMING_SNAKE_CASE					)
							elif not os.path.isfile(self.vocab_file					):
											with open(__SCREAMING_SNAKE_CASE  ,						'''wb'''					) as fi:
															a_		:  Any											=      self.sp_model.serialized_model_proto()
															fi.write(__SCREAMING_SNAKE_CASE					)

							return (out_vocab_file,)




