


from typing import List, Optional, Union

import numpy as np

from ....audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ....feature_extraction_sequence_utils import SequenceFeatureExtractor
from ....feature_extraction_utils import BatchFeature
from ....file_utils import PaddingStrategy, TensorType
from ....utils import logging


__UpperCamelCase				:  Optional[Any]      					=							logging.get_logger(__name__)





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     str			= ['input_features', 'attention_mask']





    def __init__(	self				:   int       ,   __snake_case				:   Any=80       ,   __snake_case				:   Union[str, Any]=16_000       ,   __snake_case				:   int=0.0       ,   __snake_case				:   Tuple=10       ,   __snake_case				:   Any=25       ,   __snake_case				:   Optional[int]="hamming_window"       ,   __snake_case				:   Optional[int]=32_768.0       ,   __snake_case				:   List[Any]=0.97       ,   __snake_case				:   Optional[Any]=1.0       ,   __snake_case				:   List[str]=True       ,   __snake_case				:   Union[str, Any]=True       ,   __snake_case				:   Dict=False       ,   **__snake_case				:   List[Any]       ,   ):





       '''simple docstring'''
       super().__init__(feature_size=__snake_case       ,   sampling_rate=__snake_case       ,   padding_value=__snake_case       ,   **__snake_case	)

       UpperCAmelCase_							:      Any						=   feature_size
       UpperCAmelCase_							:      List[str]						=   sampling_rate
       UpperCAmelCase_							:      str						=   padding_value
       UpperCAmelCase_							:      Optional[Any]						=   hop_length
       UpperCAmelCase_							:      Optional[int]						=   win_length
       UpperCAmelCase_							:      Union[str, Any]						=   frame_signal_scale
       UpperCAmelCase_							:      int						=   preemphasis_coeff
       UpperCAmelCase_							:      List[Any]						=   mel_floor
       UpperCAmelCase_							:      List[Any]						=   normalize_means
       UpperCAmelCase_							:      Tuple						=   normalize_vars
       UpperCAmelCase_							:      List[str]						=   win_function
       UpperCAmelCase_							:      Optional[Any]						=   return_attention_mask

       UpperCAmelCase_							:      Optional[Any]						=   win_length * sampling_rate // 1_000
       UpperCAmelCase_							:      Optional[Any]						=   hop_length * sampling_rate // 1_000

       UpperCAmelCase_							:      int						=   optimal_fft_length(self.sample_size	)
       UpperCAmelCase_							:      Union[str, Any]						=   (self.n_fft // 2) + 1





    def 					_lowerCamelCase (	self				:   int       ,   __snake_case				:   np.array	):





       '''simple docstring'''
       if self.win_function == "hamming_window":
          UpperCAmelCase_							:      List[Any]						=   window_function(window_length=self.sample_size       ,   name=self.win_function       ,   periodic=__snake_case	)
       else:
          UpperCAmelCase_							:      str						=   window_function(window_length=self.sample_size       ,   name=self.win_function	)

       UpperCAmelCase_							:      List[Any]						=   mel_filter_bank(
           num_frequency_bins=self.n_freqs       ,   num_mel_filters=self.feature_size       ,   min_frequency=0.0       ,   max_frequency=self.sampling_rate / 2.0       ,   sampling_rate=self.sampling_rate       ,   )

       UpperCAmelCase_							:      str						=   spectrogram(
           one_waveform * self.frame_signal_scale       ,   window=__snake_case       ,   frame_length=self.sample_size       ,   hop_length=self.sample_stride       ,   fft_length=self.n_fft       ,   center=__snake_case       ,   preemphasis=self.preemphasis_coeff       ,   mel_filters=__snake_case       ,   mel_floor=self.mel_floor       ,   log_mel='''log'''       ,   )
       return msfc_features.T





    def 					_lowerCamelCase (	self				:   int       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   List[str]	):





       '''simple docstring'''
       # make sure we normalize float32 arrays
       if self.normalize_means:
          UpperCAmelCase_							:      int						=   x[:input_length].mean(axis=0	)
          UpperCAmelCase_							:      str						=   np.subtract(__snake_case       ,   __snake_case	)
       if self.normalize_vars:
          UpperCAmelCase_							:      int						=   x[:input_length].std(axis=0	)
          UpperCAmelCase_							:      Optional[Any]						=   np.divide(__snake_case       ,   __snake_case	)

       if input_length < x.shape[0]:
          UpperCAmelCase_							:      Union[str, Any]						=   padding_value

       # make sure array is in float32
       UpperCAmelCase_							:      List[str]						=   x.astype(np.floataa	)

       return x





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   List[np.ndarray]       ,   __snake_case				:   Optional[np.ndarray] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   attention_mask.sum(-1	) if attention_mask is not None else [x.shape[0] for x in input_features]
       return [self._normalize_one(__snake_case       ,   __snake_case       ,   self.padding_value	) for x, n in zip(__snake_case       ,   __snake_case	)]





    def __call__(	self				:   str       ,   __snake_case				:   Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]       ,   __snake_case				:   Union[bool, str, PaddingStrategy] = False       ,   __snake_case				:   Optional[int] = None       ,   __snake_case				:   bool = False       ,   __snake_case				:   Optional[int] = None       ,   __snake_case				:   Optional[bool] = None       ,   __snake_case				:   Optional[Union[str, TensorType]] = None       ,   __snake_case				:   Optional[int] = None       ,   **__snake_case				:   Tuple       ,   ):





       '''simple docstring'''

       if sampling_rate is not None:
          if sampling_rate != self.sampling_rate:
             raise ValueError(
                 f'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
                 f''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
                 f''' {self.sampling_rate} and not {sampling_rate}.'''	)
       else:
          logger.warning(
              '''It is strongly recommended to pass the ``sampling_rate`` argument to this function. '''
              '''Failing to do so can result in silent errors that might be hard to debug.'''	)

       UpperCAmelCase_							:      Dict						=   isinstance(__snake_case       ,   np.ndarray	) and len(raw_speech.shape	) > 1
       if is_batched_numpy and len(raw_speech.shape	) > 2:
          raise ValueError(f'''Only mono-channel audio is supported for input to {self}'''	)
       UpperCAmelCase_							:      Tuple						=   is_batched_numpy or (
           isinstance(__snake_case       ,   (list, tuple)	) and (isinstance(raw_speech[0]       ,   (np.ndarray, tuple, list)	))
       )

       if is_batched:
          UpperCAmelCase_							:      str						=   [np.asarray(__snake_case       ,   dtype=np.floataa	) for speech in raw_speech]
       elif not is_batched and not isinstance(__snake_case       ,   np.ndarray	):
          UpperCAmelCase_							:      str						=   np.asarray(__snake_case       ,   dtype=np.floataa	)
       elif isinstance(__snake_case       ,   np.ndarray	) and raw_speech.dtype is np.dtype(np.floataa	):
          UpperCAmelCase_							:      Dict						=   raw_speech.astype(np.floataa	)

       # always return batch
       if not is_batched:
          UpperCAmelCase_							:      Union[str, Any]						=   [raw_speech]

       # extract fbank features
       UpperCAmelCase_							:      Any						=   [self._extract_mfsc_features(__snake_case	) for one_waveform in raw_speech]

       # convert into correct format for padding
       UpperCAmelCase_							:      Any						=   BatchFeature({'''input_features''': features}	)

       UpperCAmelCase_							:      List[Any]						=   self.pad(
           __snake_case       ,   padding=__snake_case       ,   max_length=__snake_case       ,   truncation=__snake_case       ,   pad_to_multiple_of=__snake_case       ,   return_attention_mask=__snake_case       ,   **__snake_case       ,   )
       # make sure list is in array format
       UpperCAmelCase_							:      List[str]						=   padded_inputs.get('''input_features'''	)
       if isinstance(input_features[0]       ,   __snake_case	):
          UpperCAmelCase_							:      int						=   [np.asarray(__snake_case       ,   dtype=np.floataa	) for feature in input_features]

       UpperCAmelCase_							:      List[Any]						=   padded_inputs.get('''attention_mask'''	)
       if attention_mask is not None:
          UpperCAmelCase_							:      Optional[Any]						=   [np.asarray(__snake_case       ,   dtype=np.intaa	) for array in attention_mask]

       if self.normalize_means or self.normalize_vars:
          UpperCAmelCase_							:      str						=   (
              np.array(__snake_case       ,   dtype=np.intaa	)
              if self._get_padding_strategies(__snake_case       ,   max_length=__snake_case	) is not PaddingStrategy.DO_NOT_PAD
              and padding
              else None
          )
          UpperCAmelCase_							:      str						=   self.normalize(
              padded_inputs['''input_features''']       ,   attention_mask=__snake_case	)

       if return_tensors is not None:
          UpperCAmelCase_							:      int						=   padded_inputs.convert_to_tensors(__snake_case	)

       return padded_inputs



import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_ta import TaTokenizer
else:
       __UpperCamelCase				:  Optional[int]      					=							None


__UpperCamelCase				:  Union[str, Any]      					=							logging.get_logger(__name__)

__UpperCamelCase				:  List[Any]      					=							{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

__UpperCamelCase				:  Optional[int]      					=							{
    'vocab_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/spiece.model',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/spiece.model',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/spiece.model',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/spiece.model',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/tokenizer.json',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/tokenizer.json',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/tokenizer.json',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/tokenizer.json',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/tokenizer.json',
    },
}


# TODO(PVP) - this should be removed in Transformers v5
__UpperCamelCase				:  Optional[Any]      					=							{
    't5-small': 512,
    't5-base': 512,
    't5-large': 512,
    't5-3b': 512,
    't5-11b': 512,
}





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     Optional[int]			= VOCAB_FILES_NAMES
    A_  :     List[Any]			= PRETRAINED_VOCAB_FILES_MAP
    A_  :     int			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    A_  :     str			= ['input_ids', 'attention_mask']
    A_  :     int			= TaTokenizer

    A_  :     List[int]			= []





    def __init__(	self				:   Union[str, Any]       ,   __snake_case				:   Tuple=None       ,   __snake_case				:   List[Any]=None       ,   __snake_case				:   int="</s>"       ,   __snake_case				:   List[Any]="<unk>"       ,   __snake_case				:   Dict="<pad>"       ,   __snake_case				:   Tuple=100       ,   __snake_case				:   int=None       ,   **__snake_case				:   Any       ,   ):





       '''simple docstring'''
       # Add extra_ids to the special token list
       if extra_ids > 0 and additional_special_tokens is None:
          UpperCAmelCase_							:      Optional[int]						=   [f'''<extra_id_{i}>''' for i in range(__snake_case	)]
       elif extra_ids > 0 and additional_special_tokens is not None:
          # Check that we have the right number of extra special tokens
          UpperCAmelCase_							:      Any						=   len(set(filter(lambda __snake_case	: bool('''extra_id_''' in str(__snake_case	)	)       ,   __snake_case	)	)	)
          if extra_tokens != extra_ids:
             raise ValueError(
                 f'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
                 ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
                 ''' tokens'''	)

       super().__init__(
           __snake_case       ,   tokenizer_file=__snake_case       ,   eos_token=__snake_case       ,   unk_token=__snake_case       ,   pad_token=__snake_case       ,   extra_ids=__snake_case       ,   additional_special_tokens=__snake_case       ,   **__snake_case       ,   )

       UpperCAmelCase_							:      str						=   vocab_file
       UpperCAmelCase_							:      List[str]						=   False if not self.vocab_file else True
       UpperCAmelCase_							:      Union[str, Any]						=   extra_ids





    @staticmethod
    def 					_lowerCamelCase (	__snake_case				:   Dict       ,   __snake_case				:   List[str]       ,   __snake_case				:   Tuple	):





       '''simple docstring'''
       if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
          UpperCAmelCase_							:      str						=   TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
          if init_max_model_length is not None and init_max_model_length != max_model_length:
             return init_max_model_length
          elif init_max_model_length is None:
             warnings.warn(
                 '''This tokenizer was incorrectly instantiated with a model max length of'''
                 f''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
                 ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
                 ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
                 f''' {pretrained_model_name_or_path} automatically truncating your input to'''
                 f''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
                 f''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
                 ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
                 ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''       ,   __snake_case       ,   )

       return max_model_length





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   str       ,   __snake_case				:   Optional[str] = None	):





       '''simple docstring'''
       if not self.can_save_slow_tokenizer:
          raise ValueError(
              '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
              '''tokenizer.'''	)

       if not os.path.isdir(__snake_case	):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
          return
       UpperCAmelCase_							:      str						=   os.path.join(
           __snake_case       ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

       if os.path.abspath(self.vocab_file	) != os.path.abspath(__snake_case	):
          copyfile(self.vocab_file       ,   __snake_case	)
          logger.info(f'''Copy vocab file to {out_vocab_file}'''	)

       return (out_vocab_file,)





    def 					_lowerCamelCase (	self				:   List[str]       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
          return self.prefix_tokens + token_ids_a
       else:
          UpperCAmelCase_							:      int						=   token_ids_a + [self.eos_token_id]
          return self.prefix_tokens + token_ids_a + token_ids_a





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   [self.eos_token_id]

       if token_ids_a is None:
          return len(token_ids_a + eos	) * [0]
       return len(token_ids_a + eos + token_ids_a + eos	) * [0]





    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       return list(
           set(filter(lambda __snake_case	: bool(re.search(R'''<extra_id_\d+>'''       ,   __snake_case	)	) is not None       ,   self.additional_special_tokens	)	)	)





    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       return [self.convert_tokens_to_ids(__snake_case	) for token in self.get_sentinel_tokens()]