


import copy
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


snake_case							: List[Any]      =   logging.get_logger(__name__)







class  snake_case_     (lowerCamelCase_   ):
					UpperCAmelCase__      :					str					   =    ['''input_features''']
					def __init__(   self     :Tuple    ,__snake_case     :Tuple=80    ,__snake_case     :Any=1_60_00    ,__snake_case     :Any=1_60    ,__snake_case     :List[Any]=30    ,__snake_case     :Optional[Any]=4_00    ,__snake_case     :Tuple=0.0    ,__snake_case     :Any=False    ,**__snake_case     :Optional[Any]    ,)      ->					Optional[int]:
												super().__init__(
												    feature_size=__snake_case    ,sampling_rate=__snake_case    ,padding_value=__snake_case    ,return_attention_mask=__snake_case    ,**__snake_case    ,)
												a__    =  n_fft
												a__    =  hop_length
												a__    =  chunk_length
												a__    =  chunk_length * sampling_rate
												a__    =  self.n_samples // hop_length
												a__    =  sampling_rate
												a__    =  mel_filter_bank(
												    num_frequency_bins=1 + n_fft // 2    ,num_mel_filters=__snake_case    ,min_frequency=0.0    ,max_frequency=80_00.0    ,sampling_rate=__snake_case    ,norm='slaney'    ,mel_scale='slaney'    ,)
					def 				lowerCamelCase__(   self     :Tuple    ,__snake_case     :np.array   )      ->					np.ndarray:
												a__    =  spectrogram(
												    __snake_case    ,window_function(self.n_fft    ,'hann'   )    ,frame_length=self.n_fft    ,hop_length=self.hop_length    ,power=2.0    ,mel_filters=self.mel_filters    ,log_mel='log10'    ,)
												a__    =  log_spec[:, :-1]
												a__    =  np.maximum(__snake_case    ,log_spec.max() - 8.0   )
												a__    =  (log_spec + 4.0) / 4.0
												return log_spec
					@staticmethod
					# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
					def 				lowerCamelCase__(   __snake_case     :List[np.ndarray]    ,__snake_case     :List[np.ndarray]    ,__snake_case     :float = 0.0   )      ->					List[np.ndarray]:
												if attention_mask is not None:
																			a__    =  np.array(__snake_case    ,np.intaa   )
																			a__    =  []

																			for vector, length in zip(__snake_case    ,attention_mask.sum(-1   )   ):
																										a__    =  (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1E-7   )
																										if length < normed_slice.shape[0]:
																																	a__    =  padding_value

																										normed_input_values.append(__snake_case   )
												else:
																			a__    =  [(x - x.mean()) / np.sqrt(x.var() + 1E-7   ) for x in input_values]

												return normed_input_values
					def __call__(   self     :Optional[int]    ,__snake_case     :Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]    ,__snake_case     :bool = True    ,__snake_case     :Optional[int] = None    ,__snake_case     :Optional[Union[str, TensorType]] = None    ,__snake_case     :Optional[bool] = None    ,__snake_case     :Optional[str] = "max_length"    ,__snake_case     :Optional[int] = None    ,__snake_case     :Optional[int] = None    ,__snake_case     :Optional[bool] = None    ,**__snake_case     :Dict    ,)      ->					BatchFeature:

												if sampling_rate is not None:
																			if sampling_rate != self.sampling_rate:
																										raise ValueError(
																										    F'The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a'
																										    F' sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input'
																										    F' was sampled with {self.sampling_rate} and not {sampling_rate}.'   )
												else:
																			logger.warning(
																			    'It is strongly recommended to pass the `sampling_rate` argument to this function. '
																			    'Failing to do so can result in silent errors that might be hard to debug.'   )

												a__    =  isinstance(__snake_case    ,np.ndarray   ) and len(raw_speech.shape   ) > 1
												if is_batched_numpy and len(raw_speech.shape   ) > 2:
																			raise ValueError(F'Only mono-channel audio is supported for input to {self}'   )
												a__    =  is_batched_numpy or (
												    isinstance(__snake_case    ,(list, tuple)   ) and (isinstance(raw_speech[0]    ,(np.ndarray, tuple, list)   ))
												)

												if is_batched:
																			a__    =  [np.asarray([speech]    ,dtype=np.floataa   ).T for speech in raw_speech]
												elif not is_batched and not isinstance(__snake_case    ,np.ndarray   ):
																			a__    =  np.asarray(__snake_case    ,dtype=np.floataa   )
												elif isinstance(__snake_case    ,np.ndarray   ) and raw_speech.dtype is np.dtype(np.floataa   ):
																			a__    =  raw_speech.astype(np.floataa   )

												# always return batch
												if not is_batched:
																			a__    =  [np.asarray([raw_speech]   ).T]

												a__    =  BatchFeature({'input_features': raw_speech}   )

												# convert into correct format for padding

												a__    =  self.pad(
												    __snake_case    ,padding=__snake_case    ,max_length=max_length if max_length else self.n_samples    ,truncation=__snake_case    ,pad_to_multiple_of=__snake_case    ,return_attention_mask=return_attention_mask or do_normalize    ,)

												# zero-mean and unit-variance normalization
												if do_normalize:
																			a__    =  self.zero_mean_unit_var_norm(
																			    padded_inputs['input_features']    ,attention_mask=padded_inputs['attention_mask']    ,padding_value=self.padding_value    ,)
																			a__    =  np.stack(padded_inputs['input_features']    ,axis=0   )

												# make sure list is in array format
												a__    =  padded_inputs.get('input_features'   ).transpose(2    ,0    ,1   )

												a__    =  [self._np_extract_fbank_features(__snake_case   ) for waveform in input_features[0]]

												if isinstance(input_features[0]    ,__snake_case   ):
																			a__    =  [np.asarray(__snake_case    ,dtype=np.floataa   ) for feature in input_features]
												else:
																			a__    =  input_features

												if return_attention_mask:
																			# rescale from sample (48000) to feature (3000)
																			a__    =  padded_inputs['attention_mask'][:, :: self.hop_length]

												if return_tensors is not None:
																			a__    =  padded_inputs.convert_to_tensors(__snake_case   )

												return padded_inputs



					def 				lowerCamelCase__(   self     :List[Any]   )      ->					Dict[str, Any]:
												a__    =  copy.deepcopy(self.__dict__   )
												a__    =  self.__class__.__name__
												if "mel_filters" in output:
																			del output["mel_filters"]
												return output








import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

snake_case							: Dict      =   logging.get_logger(__name__)


snake_case							: Any      =   {name: getattr(transformers, name + '''Fast''') for name in SLOW_TO_FAST_CONVERTERS}






def 							__lowercase     (				__lowerCAmelCase							:       Optional[Any]      ,					__lowerCAmelCase							:       Union[str, Any]      ,					__lowerCAmelCase							:       Any      ,					__lowerCAmelCase							:       Optional[Any]     ):
							if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
														raise ValueError(F'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()     )}.'     )

							if tokenizer_name is None:
														a__    =  TOKENIZER_CLASSES
							else:
														a__    =  {tokenizer_name: getattr(__lowerCAmelCase      ,					tokenizer_name + 'Fast'     )}

							logger.info(F'Loading tokenizer classes: {tokenizer_names}'     )

							for tokenizer_name in tokenizer_names:
														a__    =  TOKENIZER_CLASSES[tokenizer_name]

														a__    =  True
														if checkpoint_name is None:
																					a__    =  list(tokenizer_class.max_model_input_sizes.keys()     )
														else:
																					a__    =  [checkpoint_name]

														logger.info(F'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'     )

														for checkpoint in checkpoint_names:
																					logger.info(F'Loading {tokenizer_class.__class__.__name__} {checkpoint}'     )

																					# Load tokenizer
																					a__    =  tokenizer_class.from_pretrained(__lowerCAmelCase      ,					force_download=__lowerCAmelCase     )

																					# Save fast tokenizer
																					logger.info(F'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'     )

																					# For organization names we create sub-directories
																					if "/" in checkpoint:
																												a__   ,      a__    =  checkpoint.split('/'     )
																												a__    =  os.path.join(__lowerCAmelCase      ,					__lowerCAmelCase     )
																					elif add_prefix:
																												a__    =  checkpoint
																												a__    =  dump_path
																					else:
																												a__    =  None
																												a__    =  dump_path

																					logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'     )

																					if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()     )[0]:
																												a__    =  list(tokenizer.pretrained_vocab_files_map.values()     )[0][checkpoint]
																												a__    =  file_path.split(__lowerCAmelCase     )[-1][0]
																												if next_char == "/":
																																			a__    =  os.path.join(__lowerCAmelCase      ,					__lowerCAmelCase     )
																																			a__    =  None

																												logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'     )

																					a__    =  tokenizer.save_pretrained(
																					    __lowerCAmelCase      ,					legacy_format=__lowerCAmelCase      ,					filename_prefix=__lowerCAmelCase     )
																					logger.info(F'=> File names {file_names}'     )

																					for file_name in file_names:
																												if not file_name.endswith('tokenizer.json'     ):
																																			os.remove(__lowerCAmelCase     )
																																			logger.info(F'=> removing {file_name}'     )


if __name__ == "__main__":
						snake_case							: Union[str, Any]      =   argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '''--dump_path''', default=None, type=str, required=True, help='''Path to output generated fast tokenizer files.'''
						)
						parser.add_argument(
						    '''--tokenizer_name''',
						    default=None,
						    type=str,
						    help=(
						        f"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
						        '''download and convert all the checkpoints from AWS.'''
						    ),
						)
						parser.add_argument(
						    '''--checkpoint_name''',
						    default=None,
						    type=str,
						    help='''Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.''',
						)
						parser.add_argument(
						    '''--force_download''',
						    action='''store_true''',
						    help='''Re-download checkpoints.''',
						)
						snake_case							: List[str]      =   parser.parse_args()

						convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)




