


import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCamelCase				:  List[str]      					=							logging.get_logger(__name__)

__UpperCamelCase				:  List[Any]      					=							'▁'

__UpperCamelCase				:  List[str]      					=							{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
}

__UpperCamelCase				:  Dict      					=							{
    'vocab_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'
        ),
    },
    'spm_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'
        )
    },
}

__UpperCamelCase				:  str      					=							{
    'facebook/s2t-small-librispeech-asr': 1024,
}

__UpperCamelCase				:  Tuple      					=							['pt', 'fr', 'ru', 'nl', 'ro', 'it', 'es', 'de']

__UpperCamelCase				:  Union[str, Any]      					=							{'mustc': MUSTC_LANGS}





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     Dict			= VOCAB_FILES_NAMES
    A_  :     Optional[Any]			= PRETRAINED_VOCAB_FILES_MAP
    A_  :     Tuple			= MAX_MODEL_INPUT_SIZES
    A_  :     Union[str, Any]			= ['input_ids', 'attention_mask']

    A_  :     List[str]			= []





    def __init__(	self				:   int       ,   __snake_case				:   int       ,   __snake_case				:   int       ,   __snake_case				:   Optional[Any]="<s>"       ,   __snake_case				:   str="</s>"       ,   __snake_case				:   Optional[Any]="<pad>"       ,   __snake_case				:   Any="<unk>"       ,   __snake_case				:   str=False       ,   __snake_case				:   List[Any]=False       ,   __snake_case				:   Optional[int]=None       ,   __snake_case				:   Optional[int]=None       ,   __snake_case				:   int = None       ,   **__snake_case				:   List[Any]       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   {} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           bos_token=__a       ,   eos_token=__a       ,   unk_token=__a       ,   pad_token=__a       ,   do_upper_case=__a       ,   do_lower_case=__a       ,   tgt_lang=__a       ,   lang_codes=__a       ,   sp_model_kwargs=self.sp_model_kwargs       ,   **__a       ,   )
       UpperCAmelCase_							:      Optional[Any]						=   do_upper_case
       UpperCAmelCase_							:      int						=   do_lower_case

       UpperCAmelCase_							:      int						=   load_json(__a	)
       UpperCAmelCase_							:      Union[str, Any]						=   {v: k for k, v in self.encoder.items()}
       UpperCAmelCase_							:      Dict						=   spm_file
       UpperCAmelCase_							:      Tuple						=   load_spm(__a       ,   self.sp_model_kwargs	)

       if lang_codes is not None:
          UpperCAmelCase_							:      str						=   lang_codes
          UpperCAmelCase_							:      int						=   LANGUAGES[lang_codes]
          UpperCAmelCase_							:      Optional[Any]						=   [f'''<lang:{lang}>''' for lang in self.langs]
          UpperCAmelCase_							:      int						=   {lang: self.sp_model.PieceToId(f'''<lang:{lang}>'''	) for lang in self.langs}

          UpperCAmelCase_							:      int						=   self.lang_tokens
          UpperCAmelCase_							:      Dict						=   tgt_lang if tgt_lang is not None else self.langs[0]

          self.set_tgt_lang_special_tokens(self._tgt_lang	)
       else:
          UpperCAmelCase_							:      Optional[Any]						=   {}





    @property
    def 					_lowerCamelCase (	self				:   str	):





       '''simple docstring'''
       return len(self.encoder	)





    @property
    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       return self._tgt_lang





    @tgt_lang.setter
    def 					_lowerCamelCase (	self				:   int       ,   __snake_case				:   Tuple	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   new_tgt_lang
       self.set_tgt_lang_special_tokens(__a	)





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   List[str]	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   self.lang_code_to_id[tgt_lang]
       UpperCAmelCase_							:      List[Any]						=   [lang_code_id]





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   Union[str, Any]	):





       '''simple docstring'''
       return self.sp_model.encode(__a       ,   out_type=__a	)





    def 					_lowerCamelCase (	self				:   Tuple       ,   __snake_case				:   Optional[int]	):





       '''simple docstring'''
       return self.encoder.get(__a       ,   self.encoder[self.unk_token]	)





    def 					_lowerCamelCase (	self				:   Tuple       ,   __snake_case				:   int	):





       '''simple docstring'''
       return self.decoder.get(__a       ,   self.unk_token	)





    def 					_lowerCamelCase (	self				:   Any       ,   __snake_case				:   Union[str, Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   []
       UpperCAmelCase_							:      int						=   ''''''
       for token in tokens:
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
             UpperCAmelCase_							:      Any						=   self.sp_model.decode(__a	)
             out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
             UpperCAmelCase_							:      str						=   []
          else:
             current_sub_tokens.append(__a	)
       UpperCAmelCase_							:      List[str]						=   self.sp_model.decode(__a	)
       out_string += decoded.upper() if self.do_upper_case else decoded
       return out_string.strip()





    def 					_lowerCamelCase (	self				:   int       ,   __snake_case				:   List[Any]       ,   __snake_case				:   Tuple=None	):





       '''simple docstring'''
       if token_ids_a is None:
          return self.prefix_tokens + token_ids_a + [self.eos_token_id]
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]





    def 					_lowerCamelCase (	self				:   Tuple       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   str = None       ,   __snake_case				:   Union[str, Any] = False	):





       '''simple docstring'''

       if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__a       ,   token_ids_a=__a       ,   already_has_special_tokens=__a	)

       UpperCAmelCase_							:      Tuple						=   [1] * len(self.prefix_tokens	)
       UpperCAmelCase_							:      List[Any]						=   [1]
       if token_ids_a is None:
          return prefix_ones + ([0] * len(__a	)) + suffix_ones
       return prefix_ones + ([0] * len(__a	)) + ([0] * len(__a	)) + suffix_ones





    def 					_lowerCamelCase (	self				:   Union[str, Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   self.encoder.copy()
       vocab.update(self.added_tokens_encoder	)
       return vocab





    def __getstate__(	self				:   List[str]	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   self.__dict__.copy()
       UpperCAmelCase_							:      Dict						=   None
       return state





    def __setstate__(	self				:   Optional[Any]       ,   __snake_case				:   Union[str, Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[int]						=   d

       # for backward compatibility
       if not hasattr(self       ,   '''sp_model_kwargs'''	):
          UpperCAmelCase_							:      Optional[Any]						=   {}

       UpperCAmelCase_							:      Optional[Any]						=   load_spm(self.spm_file       ,   self.sp_model_kwargs	)





    def 					_lowerCamelCase (	self				:   Tuple       ,   __snake_case				:   List[str]       ,   __snake_case				:   Tuple = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   Path(__a	)
       assert save_dir.is_dir(), f'''{save_directory} should be a directory'''
       UpperCAmelCase_							:      int						=   save_dir / (
           (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
       )
       UpperCAmelCase_							:      Optional[Any]						=   save_dir / (
           (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
       )

       save_json(self.encoder       ,   __a	)

       if os.path.abspath(self.spm_file	) != os.path.abspath(__a	) and os.path.isfile(self.spm_file	):
          copyfile(self.spm_file       ,   __a	)
       elif not os.path.isfile(self.spm_file	):
          with open(__a       ,   '''wb'''	) as fi:
             UpperCAmelCase_							:      int						=   self.sp_model.serialized_model_proto()
             fi.write(__a	)

       return (str(__a	), str(__a	))





def      snake_case_						( __lowercase			,       __lowercase       )     ->   sentencepiece.SentencePieceProcessor:
   UpperCAmelCase_							:      Optional[Any]						=   sentencepiece.SentencePieceProcessor(**__snake_case       )
   spm.Load(str(__snake_case       )       )
   return spm
def      snake_case_						( __lowercase       )     ->   Union[Dict, List]:





   with open(__snake_case			,       '''r'''       ) as f:
      return json.load(__snake_case       )


def      snake_case_						( __lowercase			,       __lowercase       )     ->   None:





   with open(__snake_case			,       '''w'''       ) as f:
      json.dump(__snake_case			,       __snake_case			,       indent=2       )



from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase				:  str      					=							logging.get_logger(__name__)

__UpperCamelCase				:  Any      					=							{
    'tiiuae/falcon-40b': 'https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json',
    'tiiuae/falcon-7b': 'https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json',
}





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     int			= 'falcon'
    A_  :     int			= ['past_key_values']





    def __init__(	self				:   Optional[Any]       ,   __snake_case				:   Tuple=65_024       ,   __snake_case				:   List[str]=4_544       ,   __snake_case				:   Optional[Any]=32       ,   __snake_case				:   Any=71       ,   __snake_case				:   str=1E-5       ,   __snake_case				:   List[str]=0.02       ,   __snake_case				:   List[Any]=True       ,   __snake_case				:   Dict=0.0       ,   __snake_case				:   Optional[Any]=0.0       ,   __snake_case				:   Any=None       ,   __snake_case				:   List[Any]=False       ,   __snake_case				:   Dict=False       ,   __snake_case				:   Optional[int]=True       ,   __snake_case				:   List[Any]=True       ,   __snake_case				:   Optional[Any]=False       ,   __snake_case				:   Dict=11       ,   __snake_case				:   List[str]=11       ,   **__snake_case				:   int       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   vocab_size
       # Backward compatibility with n_embed kwarg
       UpperCAmelCase_							:      Union[str, Any]						=   kwargs.pop('''n_embed'''       ,   __snake_case	)
       UpperCAmelCase_							:      str						=   hidden_size if n_embed is None else n_embed
       UpperCAmelCase_							:      Tuple						=   num_hidden_layers
       UpperCAmelCase_							:      Tuple						=   num_attention_heads
       UpperCAmelCase_							:      Optional[int]						=   layer_norm_epsilon
       UpperCAmelCase_							:      int						=   initializer_range
       UpperCAmelCase_							:      Optional[int]						=   use_cache
       UpperCAmelCase_							:      List[Any]						=   hidden_dropout
       UpperCAmelCase_							:      Any						=   attention_dropout

       UpperCAmelCase_							:      Tuple						=   bos_token_id
       UpperCAmelCase_							:      List[Any]						=   eos_token_id
       UpperCAmelCase_							:      Any						=   num_attention_heads if num_kv_heads is None else num_kv_heads
       UpperCAmelCase_							:      Optional[int]						=   alibi
       UpperCAmelCase_							:      Dict						=   new_decoder_architecture
       UpperCAmelCase_							:      List[Any]						=   multi_query  # Ignored when new_decoder_architecture is True
       UpperCAmelCase_							:      Tuple						=   parallel_attn
       UpperCAmelCase_							:      List[Any]						=   bias

       super().__init__(bos_token_id=__snake_case       ,   eos_token_id=__snake_case       ,   **__snake_case	)





    @property
    def 					_lowerCamelCase (	self				:   Optional[int]	):





       '''simple docstring'''
       return self.hidden_size // self.num_attention_heads





    @property
    def 					_lowerCamelCase (	self				:   Optional[int]	):





       '''simple docstring'''
       return not self.alibi