



"""simple docstring"""






# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
	import torch



class     A__     (						TensorFormatter[Mapping, 'torch.Tensor', Mapping]							):




	'''simple docstring'''





	def __init__(  self:	Any ,    _SCREAMING_SNAKE_CASE:	Optional[Any]=None ,    **_SCREAMING_SNAKE_CASE:	str)       -> str:

				"""simple docstring"""

				super().__init__(features=_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Union[str, Any]				= torch_tensor_kwargs
				import torch  # noqa import torch at initialization

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any] ,    _SCREAMING_SNAKE_CASE:	Union[str, Any])       -> Union[str, Any]:

				"""simple docstring"""

				import torch

				if isinstance(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE) and column:
							if all(
							    isinstance(_SCREAMING_SNAKE_CASE ,    torch.Tensor) and x.shape == column[0].shape and x.dtype == column[0].dtype
							    for x in column):
										return torch.stack(_SCREAMING_SNAKE_CASE)
				return column

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict ,    _SCREAMING_SNAKE_CASE:	Optional[Any])       -> Tuple:

				"""simple docstring"""

				import torch

				if isinstance(_SCREAMING_SNAKE_CASE ,    (str, bytes, type(_SCREAMING_SNAKE_CASE))):
							return value
				elif isinstance(_SCREAMING_SNAKE_CASE ,    (np.character, np.ndarray)) and np.issubdtype(value.dtype ,    np.character):
							return value.tolist()

				__lowerCAmelCase   :						Union[str, Any]				= {}

				if isinstance(_SCREAMING_SNAKE_CASE ,    (np.number, np.ndarray)) and np.issubdtype(value.dtype ,    np.integer):
							__lowerCAmelCase   :						Optional[Any]				= {"dtype": torch.intaa}
				elif isinstance(_SCREAMING_SNAKE_CASE ,    (np.number, np.ndarray)) and np.issubdtype(value.dtype ,    np.floating):
							__lowerCAmelCase   :						str				= {"dtype": torch.floataa}
				elif config.PIL_AVAILABLE and "PIL" in sys.modules:
							import PIL.Image

							if isinstance(_SCREAMING_SNAKE_CASE ,    PIL.Image.Image):
										__lowerCAmelCase   :						Any				= np.asarray(_SCREAMING_SNAKE_CASE)
				return torch.tensor(_SCREAMING_SNAKE_CASE ,    **{**default_dtype, **self.torch_tensor_kwargs})

	def 							_SCREAMING_SNAKE_CASE     (  self:	Tuple ,    _SCREAMING_SNAKE_CASE:	int)       -> Optional[int]:

				"""simple docstring"""

				import torch

				# support for torch, tf, jax etc.
				if hasattr(_SCREAMING_SNAKE_CASE ,    "__array__") and not isinstance(_SCREAMING_SNAKE_CASE ,    torch.Tensor):
							__lowerCAmelCase   :						Any				= data_struct.__array__()
				# support for nested types like struct of list of struct
				if isinstance(_SCREAMING_SNAKE_CASE ,    np.ndarray):
							if data_struct.dtype == object:  # torch tensors cannot be instantied from an array of objects
										return self._consolidate([self.recursive_tensorize(_SCREAMING_SNAKE_CASE) for substruct in data_struct])
				elif isinstance(_SCREAMING_SNAKE_CASE ,    (list, tuple)):
							return self._consolidate([self.recursive_tensorize(_SCREAMING_SNAKE_CASE) for substruct in data_struct])
				return self._tensorize(_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any] ,    _SCREAMING_SNAKE_CASE:	dict)       -> Optional[Any]:

				"""simple docstring"""

				return map_nested(self._recursive_tensorize ,    _SCREAMING_SNAKE_CASE ,    map_list=_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any ,    _SCREAMING_SNAKE_CASE:	pa.Table)       -> Mapping:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= self.numpy_arrow_extractor().extract_row(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Any				= self.python_features_decoder.decode_row(_SCREAMING_SNAKE_CASE)
				return self.recursive_tensorize(_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict ,    _SCREAMING_SNAKE_CASE:	pa.Table)       -> "torch.Tensor":

				"""simple docstring"""

				__lowerCAmelCase   :						int				= self.numpy_arrow_extractor().extract_column(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Optional[int]				= self.python_features_decoder.decode_column(_SCREAMING_SNAKE_CASE ,    pa_table.column_names[0])
				__lowerCAmelCase   :						Optional[int]				= self.recursive_tensorize(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Union[str, Any]				= self._consolidate(_SCREAMING_SNAKE_CASE)
				return column







	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any] ,    _SCREAMING_SNAKE_CASE:	pa.Table)       -> Mapping:

				"""simple docstring"""

				__lowerCAmelCase   :						Optional[int]				= self.numpy_arrow_extractor().extract_batch(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						List[str]				= self.python_features_decoder.decode_batch(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						str				= self.recursive_tensorize(_SCREAMING_SNAKE_CASE)
				for column_name in batch:
							__lowerCAmelCase   :						List[Any]				= self._consolidate(batch[column_name])
				return batch




"""simple docstring"""






import itertools
import random
import unittest

import numpy as np

from transformers import is_speech_available
from transformers.testing_utils import require_torch, require_torchaudio

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
	from transformers import SpeechaTextFeatureExtractor

__snake_case							:     Union[str, Any]   =					random.Random()


def     _lowercase       (						__snake_case      ,__snake_case=1.0      ,__snake_case=None      ,__snake_case=None			)		->     List[Any]:
			if rng is None:
						__lowerCAmelCase   :						Dict				= global_rng

			__lowerCAmelCase   :						Tuple				= []
			for batch_idx in range(shape[0]			):
						values.append([]			)
						for _ in range(shape[1]			):
									values[-1].append(rng.random() * scale			)

			return values



@require_torch
@require_torchaudio
class     A__     (						unittest.TestCase							):




	'''simple docstring'''





	def __init__(  self:	Optional[Any] ,    _SCREAMING_SNAKE_CASE:	Any ,    _SCREAMING_SNAKE_CASE:	List[Any]=7 ,    _SCREAMING_SNAKE_CASE:	int=400 ,    _SCREAMING_SNAKE_CASE:	List[str]=2000 ,    _SCREAMING_SNAKE_CASE:	Optional[Any]=24 ,    _SCREAMING_SNAKE_CASE:	Dict=24 ,    _SCREAMING_SNAKE_CASE:	Optional[int]=0.0 ,    _SCREAMING_SNAKE_CASE:	Any=1_6000 ,    _SCREAMING_SNAKE_CASE:	int=True ,    _SCREAMING_SNAKE_CASE:	Union[str, Any]=True ,    )       -> List[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						Any				= parent
				__lowerCAmelCase   :						str				= batch_size
				__lowerCAmelCase   :						List[str]				= min_seq_length
				__lowerCAmelCase   :						Any				= max_seq_length
				__lowerCAmelCase   :						Dict				= (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
				__lowerCAmelCase   :						Union[str, Any]				= feature_size
				__lowerCAmelCase   :						int				= num_mel_bins
				__lowerCAmelCase   :						Optional[Any]				= padding_value
				__lowerCAmelCase   :						List[Any]				= sampling_rate
				__lowerCAmelCase   :						Dict				= return_attention_mask
				__lowerCAmelCase   :						int				= do_normalize

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Optional[int]:

				"""simple docstring"""

				return {
				    "feature_size": self.feature_size,
				    "num_mel_bins": self.num_mel_bins,
				    "padding_value": self.padding_value,
				    "sampling_rate": self.sampling_rate,
				    "return_attention_mask": self.return_attention_mask,
				    "do_normalize": self.do_normalize,
				}







	def 							_SCREAMING_SNAKE_CASE     (  self:	Tuple ,    _SCREAMING_SNAKE_CASE:	Tuple=False ,    _SCREAMING_SNAKE_CASE:	int=False)       -> Optional[int]:

				"""simple docstring"""

				def _flatten(_SCREAMING_SNAKE_CASE:	Optional[Any]):
							return list(itertools.chain(*_SCREAMING_SNAKE_CASE))

				if equal_length:
							__lowerCAmelCase   :						Tuple				= [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
				else:
							# make sure that inputs increase in size
							__lowerCAmelCase   :						List[Any]				= [
							    floats_list((x, self.feature_size))
							    for x in range(self.min_seq_length ,    self.max_seq_length ,    self.seq_length_diff)
							]
				if numpify:
							__lowerCAmelCase   :						Any				= [np.asarray(_SCREAMING_SNAKE_CASE) for x in speech_inputs]
				return speech_inputs






@require_torch
@require_torchaudio
class     A__     (						__SCREAMING_SNAKE_CASE						,	unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE					   =			SpeechaTextFeatureExtractor if is_speech_available() else None

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> Tuple:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= SpeechaTextFeatureExtractionTester(self)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[Any] ,    _SCREAMING_SNAKE_CASE:	Optional[Any])       -> str:

				"""simple docstring"""

				self.assertTrue(np.all(np.mean(_SCREAMING_SNAKE_CASE ,    axis=0) < 1e-3))
				self.assertTrue(np.all(np.abs(np.var(_SCREAMING_SNAKE_CASE ,    axis=0) - 1) < 1e-3))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any])       -> int:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				# create three inputs of length 800, 1000, and 1200
				__lowerCAmelCase   :						Tuple				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						Optional[int]				= [np.asarray(_SCREAMING_SNAKE_CASE) for speech_input in speech_inputs]

				# Test feature size
				__lowerCAmelCase   :						Optional[int]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    padding=_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				self.assertTrue(input_features.ndim == 3)
				self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size)

				# Test not batched input
				__lowerCAmelCase   :						Optional[Any]				= feature_extractor(speech_inputs[0] ,    return_tensors="np").input_features
				__lowerCAmelCase   :						Any				= feature_extractor(np_speech_inputs[0] ,    return_tensors="np").input_features
				self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    atol=1e-3))

				# Test batched
				__lowerCAmelCase   :						Tuple				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				__lowerCAmelCase   :						Tuple				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				for enc_seq_a, enc_seq_a in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    atol=1e-3))

				# Test 2-D numpy arrays are batched.
				__lowerCAmelCase   :						List[str]				= [floats_list((1, x))[0] for x in (800, 800, 800)]
				__lowerCAmelCase   :						Dict				= np.asarray(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Optional[int]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				__lowerCAmelCase   :						Optional[Any]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				for enc_seq_a, enc_seq_a in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    atol=1e-3))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any])       -> List[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						Tuple				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]

				__lowerCAmelCase   :						Optional[Any]				= ["longest", "max_length", "do_not_pad"]
				__lowerCAmelCase   :						str				= [None, 16, None]
				for max_length, padding in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							__lowerCAmelCase   :						str				= feature_extractor(
							    _SCREAMING_SNAKE_CASE ,    padding=_SCREAMING_SNAKE_CASE ,    max_length=_SCREAMING_SNAKE_CASE ,    return_attention_mask=_SCREAMING_SNAKE_CASE)
							__lowerCAmelCase   :						Optional[Any]				= inputs.input_features
							__lowerCAmelCase   :						Optional[Any]				= inputs.attention_mask
							__lowerCAmelCase   :						Tuple				= [np.sum(_SCREAMING_SNAKE_CASE) for x in attention_mask]

							self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]])
							self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]])
							self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						Any				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]

				__lowerCAmelCase   :						Dict				= ["longest", "max_length", "do_not_pad"]
				__lowerCAmelCase   :						List[Any]				= [None, 16, None]
				for max_length, padding in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							__lowerCAmelCase   :						List[str]				= feature_extractor(
							    _SCREAMING_SNAKE_CASE ,    max_length=_SCREAMING_SNAKE_CASE ,    padding=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE)
							__lowerCAmelCase   :						List[str]				= inputs.input_features
							__lowerCAmelCase   :						Dict				= inputs.attention_mask
							__lowerCAmelCase   :						Dict				= [np.sum(_SCREAMING_SNAKE_CASE) for x in attention_mask]

							self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]])
							self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1e-6)
							self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]])
							self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1e-6)
							self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]])

	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> Any:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						str				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						str				= feature_extractor(
				    _SCREAMING_SNAKE_CASE ,    padding="max_length" ,    max_length=4 ,    truncation=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    )
				__lowerCAmelCase   :						List[str]				= inputs.input_features
				__lowerCAmelCase   :						Dict				= inputs.attention_mask
				__lowerCAmelCase   :						Any				= np.sum(attention_mask == 1 ,    axis=1)

				self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
				self._check_zero_mean_unit_variance(input_features[1])
				self._check_zero_mean_unit_variance(input_features[2])

	def 							_SCREAMING_SNAKE_CASE     (  self:	int)       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						List[Any]				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						Any				= feature_extractor(
				    _SCREAMING_SNAKE_CASE ,    padding="longest" ,    max_length=4 ,    truncation=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    )
				__lowerCAmelCase   :						List[str]				= inputs.input_features
				__lowerCAmelCase   :						Optional[Any]				= inputs.attention_mask
				__lowerCAmelCase   :						Optional[Any]				= np.sum(attention_mask == 1 ,    axis=1)

				self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
				self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]])
				self._check_zero_mean_unit_variance(input_features[2])

				# make sure that if max_length < longest -> then pad to max_length
				self.assertEqual(input_features.shape ,    (3, 4, 24))

				__lowerCAmelCase   :						List[str]				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						List[str]				= feature_extractor(
				    _SCREAMING_SNAKE_CASE ,    padding="longest" ,    max_length=16 ,    truncation=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    )
				__lowerCAmelCase   :						Optional[Any]				= inputs.input_features
				__lowerCAmelCase   :						Optional[int]				= inputs.attention_mask
				__lowerCAmelCase   :						str				= np.sum(attention_mask == 1 ,    axis=1)

				self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
				self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]])
				self._check_zero_mean_unit_variance(input_features[2])

				# make sure that if max_length < longest -> then pad to max_length
				self.assertEqual(input_features.shape ,    (3, 6, 24))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Dict:

				"""simple docstring"""

				import torch

				__lowerCAmelCase   :						Dict				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						List[str]				= np.random.rand(100 ,    32).astype(np.floataa)
				__lowerCAmelCase   :						List[Any]				= np_speech_inputs.tolist()

				for inputs in [py_speech_inputs, np_speech_inputs]:
							__lowerCAmelCase   :						Any				= feature_extractor.pad([{"input_features": inputs}] ,    return_tensors="np")
							self.assertTrue(np_processed.input_features.dtype == np.floataa)
							__lowerCAmelCase   :						List[Any]				= feature_extractor.pad([{"input_features": inputs}] ,    return_tensors="pt")
							self.assertTrue(pt_processed.input_features.dtype == torch.floataa)

	def 							_SCREAMING_SNAKE_CASE     (  self:	str ,    _SCREAMING_SNAKE_CASE:	List[Any])       -> Optional[Any]:

				"""simple docstring"""

				from datasets import load_dataset

				__lowerCAmelCase   :						Any				= load_dataset("hf-internal-testing/librispeech_asr_dummy" ,    "clean" ,    split="validation")
				# automatic decoding with librispeech
				__lowerCAmelCase   :						List[Any]				= ds.sort("id").select(range(_SCREAMING_SNAKE_CASE))[:num_samples]["audio"]

				return [x["array"] for x in speech_samples]







	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> Any:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= np.array([
				    -1.5745, -1.7713, -1.7020, -1.6069, -1.2250, -1.1105, -0.9072, -0.8241,
				    -1.2310, -0.8098, -0.3320, -0.4101, -0.7985, -0.4996, -0.8213, -0.9128,
				    -1.0420, -1.1286, -1.0440, -0.7999, -0.8405, -1.2275, -1.5443, -1.4625,
				])
				# fmt: on

				__lowerCAmelCase   :						str				= self._load_datasamples(1)
				__lowerCAmelCase   :						Any				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						List[str]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="pt").input_features
				self.assertEquals(input_features.shape ,    (1, 584, 24))
				self.assertTrue(np.allclose(input_features[0, 0, :30] ,    _SCREAMING_SNAKE_CASE ,    atol=1e-4))