



"""simple docstring"""






import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case							:     List[str]   =					get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
	from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__snake_case							:     Union[str, Any]   =					256_047
__snake_case							:     str   =					256_145



@require_sentencepiece
@require_tokenizers
class     A__     (						__SCREAMING_SNAKE_CASE						,	unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE					   =			NllbTokenizer
	SCREAMING_SNAKE_CASE					   =			NllbTokenizerFast
	SCREAMING_SNAKE_CASE					   =			True
	SCREAMING_SNAKE_CASE					   =			True
	SCREAMING_SNAKE_CASE					   =			{}

	def 							_SCREAMING_SNAKE_CASE     (  self:	int)       -> List[str]:

				"""simple docstring"""

				super().setUp()

				# We have a SentencePiece fixture for testing
				__lowerCAmelCase   :						Dict				= NllbTokenizer(_SCREAMING_SNAKE_CASE ,    keep_accents=_SCREAMING_SNAKE_CASE)
				tokenizer.save_pretrained(self.tmpdirname)

	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						Optional[int]				= NllbTokenizer(_SCREAMING_SNAKE_CASE ,    keep_accents=_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase   :						int				= tokenizer.tokenize("This is a test")
				self.assertListEqual(_SCREAMING_SNAKE_CASE ,    ["▁This", "▁is", "▁a", "▁t", "est"])

				self.assertListEqual(
				    tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE) ,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]] ,    )

				__lowerCAmelCase   :						List[Any]				= tokenizer.tokenize("I was born in 92000, and this is falsé.")
				self.assertListEqual(
				    _SCREAMING_SNAKE_CASE ,    [
				        SPIECE_UNDERLINE + "I",
				        SPIECE_UNDERLINE + "was",
				        SPIECE_UNDERLINE + "b",
				        "or",
				        "n",
				        SPIECE_UNDERLINE + "in",
				        SPIECE_UNDERLINE + "",
				        "9",
				        "2",
				        "0",
				        "0",
				        "0",
				        ",",
				        SPIECE_UNDERLINE + "and",
				        SPIECE_UNDERLINE + "this",
				        SPIECE_UNDERLINE + "is",
				        SPIECE_UNDERLINE + "f",
				        "al",
				        "s",
				        "é",
				        ".",
				    ] ,    )
				__lowerCAmelCase   :						Dict				= tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE)
				self.assertListEqual(
				    _SCREAMING_SNAKE_CASE ,    [
				        value + tokenizer.fairseq_offset
				        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
				    ] ,    )

				__lowerCAmelCase   :						Any				= tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE)
				self.assertListEqual(
				    _SCREAMING_SNAKE_CASE ,    [
				        SPIECE_UNDERLINE + "I",
				        SPIECE_UNDERLINE + "was",
				        SPIECE_UNDERLINE + "b",
				        "or",
				        "n",
				        SPIECE_UNDERLINE + "in",
				        SPIECE_UNDERLINE + "",
				        "<unk>",
				        "2",
				        "0",
				        "0",
				        "0",
				        ",",
				        SPIECE_UNDERLINE + "and",
				        SPIECE_UNDERLINE + "this",
				        SPIECE_UNDERLINE + "is",
				        SPIECE_UNDERLINE + "f",
				        "al",
				        "s",
				        "<unk>",
				        ".",
				    ] ,    )

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
										__lowerCAmelCase   :						Dict				= self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						int				= self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Optional[Any]				= tempfile.mkdtemp()

										__lowerCAmelCase   :						int				= tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Optional[Any]				= tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE)

										# Checks it save with the same files + the tokenizer.json file for the fast one
										self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files))
										__lowerCAmelCase   :						Union[str, Any]				= tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f)
										self.assertSequenceEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)

										# Checks everything loads correctly in the same way
										__lowerCAmelCase   :						List[Any]				= tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Optional[Any]				= tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE)

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
													self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE))

										shutil.rmtree(_SCREAMING_SNAKE_CASE)

										# Save tokenizer rust, legacy_format=True
										__lowerCAmelCase   :						List[Any]				= tempfile.mkdtemp()

										__lowerCAmelCase   :						Union[str, Any]				= tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE ,    legacy_format=_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Optional[int]				= tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE)

										# Checks it save with the same files
										self.assertSequenceEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)

										# Checks everything loads correctly in the same way
										__lowerCAmelCase   :						List[str]				= tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Dict				= tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE)

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
													self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE))

										shutil.rmtree(_SCREAMING_SNAKE_CASE)

										# Save tokenizer rust, legacy_format=False
										__lowerCAmelCase   :						Any				= tempfile.mkdtemp()

										__lowerCAmelCase   :						List[Any]				= tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE ,    legacy_format=_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Tuple				= tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE)

										# Checks it saved the tokenizer.json file
										self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files))

										# Checks everything loads correctly in the same way
										__lowerCAmelCase   :						Optional[int]				= tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Any				= tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE)

										# Check special tokens are set accordingly on Rust and Python
										for key in tokenizer_pp.special_tokens_map:
													self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE))

										shutil.rmtree(_SCREAMING_SNAKE_CASE)

	@require_torch
	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> List[Any]:

				"""simple docstring"""

				if not self.test_seqaseq:
							return

				__lowerCAmelCase   :						Optional[int]				= self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(F"""{tokenizer.__class__.__name__}"""):
										# Longer text that will definitely require truncation.
										__lowerCAmelCase   :						int				= [
										    " UN Chief Says There Is No Military Solution in Syria",
										    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"
										    " Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"
										    " will only worsen the violence and misery for millions of people.",
										]
										__lowerCAmelCase   :						Optional[Any]				= [
										    "Şeful ONU declară că nu există o soluţie militară în Siria",
										    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
										    " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
										    " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
										]
										try:
													__lowerCAmelCase   :						Tuple				= tokenizer.prepare_seqaseq_batch(
													    src_texts=_SCREAMING_SNAKE_CASE ,    tgt_texts=_SCREAMING_SNAKE_CASE ,    max_length=3 ,    max_target_length=10 ,    return_tensors="pt" ,    src_lang="eng_Latn" ,    tgt_lang="ron_Latn" ,    )
										except NotImplementedError:
													return
										self.assertEqual(batch.input_ids.shape[1] ,    3)
										self.assertEqual(batch.labels.shape[1] ,    10)
										# max_target_length will default to max_length if not specified
										__lowerCAmelCase   :						Union[str, Any]				= tokenizer.prepare_seqaseq_batch(
										    _SCREAMING_SNAKE_CASE ,    tgt_texts=_SCREAMING_SNAKE_CASE ,    max_length=3 ,    return_tensors="pt")
										self.assertEqual(batch.input_ids.shape[1] ,    3)
										self.assertEqual(batch.labels.shape[1] ,    3)

										__lowerCAmelCase   :						str				= tokenizer.prepare_seqaseq_batch(
										    src_texts=_SCREAMING_SNAKE_CASE ,    max_length=3 ,    max_target_length=10 ,    return_tensors="pt")
										self.assertEqual(batch_encoder_only.input_ids.shape[1] ,    3)
										self.assertEqual(batch_encoder_only.attention_mask.shape[1] ,    3)
										self.assertNotIn("decoder_input_ids" ,    _SCREAMING_SNAKE_CASE)

	@unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece.")
	def 							_SCREAMING_SNAKE_CASE     (  self:	Tuple)       -> Optional[int]:

				"""simple docstring"""

				pass







	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[Any])       -> Optional[int]:

				"""simple docstring"""

				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
										__lowerCAmelCase   :						Optional[Any]				= [AddedToken("<special>" ,    lstrip=_SCREAMING_SNAKE_CASE)]

										__lowerCAmelCase   :						int				= self.rust_tokenizer_class.from_pretrained(
										    _SCREAMING_SNAKE_CASE ,    additional_special_tokens=_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						int				= tokenizer_r.encode("Hey this is a <special> token")

										__lowerCAmelCase   :						List[str]				= tokenizer_r.encode("<special>" ,    add_special_tokens=_SCREAMING_SNAKE_CASE)[0]

										self.assertTrue(special_token_id in r_output)

										if self.test_slow_tokenizer:
													__lowerCAmelCase   :						Optional[Any]				= self.rust_tokenizer_class.from_pretrained(
													    _SCREAMING_SNAKE_CASE ,    additional_special_tokens=_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE ,    )
													__lowerCAmelCase   :						Optional[Any]				= self.tokenizer_class.from_pretrained(
													    _SCREAMING_SNAKE_CASE ,    additional_special_tokens=_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)

													__lowerCAmelCase   :						Optional[Any]				= tokenizer_p.encode("Hey this is a <special> token")

													__lowerCAmelCase   :						Optional[Any]				= tokenizer_cr.encode("Hey this is a <special> token")

													self.assertEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)
													self.assertEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)
													self.assertTrue(special_token_id in p_output)
													self.assertTrue(special_token_id in cr_output)






@require_torch
@require_sentencepiece
@require_tokenizers
class     A__     (						unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE					   =			'facebook/nllb-200-distilled-600M'
	SCREAMING_SNAKE_CASE					   =			[
	    ' UN Chief Says There Is No Military Solution in Syria',
	    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
	]
	SCREAMING_SNAKE_CASE					   =			[
	    'Şeful ONU declară că nu există o soluţie militară în Siria',
	    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
	    ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
	    ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
	]
	SCREAMING_SNAKE_CASE					   =			[
	    2_5_6_0_4_7,
	    1_6_2_9_7,
	    1_3_4_4_0_8,
	    8_1_6_5,
	    2_4_8_0_6_6,
	    1_4_7_3_4,
	    9_5_0,
	    1_1_3_5,
	    1_0_5_7_2_1,
	    3_5_7_3,
	    8_3,
	    2_7_3_5_2,
	    1_0_8,
	    4_9_4_8_6,
	    2,
	]

	@classmethod
	def 							_SCREAMING_SNAKE_CASE     (  cls:	Any)       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						NllbTokenizer				= NllbTokenizer.from_pretrained(
				    cls.checkpoint_name ,    src_lang="eng_Latn" ,    tgt_lang="ron_Latn")
				__lowerCAmelCase   :						Union[str, Any]				= 1
				return cls

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any])       -> Tuple:

				"""simple docstring"""

				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"] ,    25_6001)
				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"] ,    25_6002)
				self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"] ,    25_6057)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Tuple)       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
				self.assertListEqual(self.expected_src_tokens ,    _SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> int:

				"""simple docstring"""

				self.assertIn(_SCREAMING_SNAKE_CASE ,    self.tokenizer.all_special_ids)
				# fmt: off
				__lowerCAmelCase   :						str				= [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
				# fmt: on

				__lowerCAmelCase   :						Dict				= self.tokenizer.decode(_SCREAMING_SNAKE_CASE ,    skip_special_tokens=_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						List[str]				= self.tokenizer.decode(generated_ids[1:] ,    skip_special_tokens=_SCREAMING_SNAKE_CASE)
				self.assertEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)
				self.assertNotIn(self.tokenizer.eos_token ,    _SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= ["this is gunna be a long sentence " * 20]
				assert isinstance(src_text[0] ,    _SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						List[str]				= 10
				__lowerCAmelCase   :						Dict				= self.tokenizer(_SCREAMING_SNAKE_CASE ,    max_length=_SCREAMING_SNAKE_CASE ,    truncation=_SCREAMING_SNAKE_CASE).input_ids[0]
				self.assertEqual(ids[-1] ,    2)
				self.assertEqual(ids[0] ,    _SCREAMING_SNAKE_CASE)
				self.assertEqual(len(_SCREAMING_SNAKE_CASE) ,    _SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[int])       -> Dict:

				"""simple docstring"""

				self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"]) ,    [25_6203, 3])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[int])       -> Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase   :						Any				= tempfile.mkdtemp()
				__lowerCAmelCase   :						int				= self.tokenizer.fairseq_tokens_to_ids
				self.tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						int				= NllbTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE)
				self.assertDictEqual(new_tok.fairseq_tokens_to_ids ,    _SCREAMING_SNAKE_CASE)

	@require_torch
	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						Any				= self.tokenizer(
				    self.src_text ,    text_target=self.tgt_text ,    padding=_SCREAMING_SNAKE_CASE ,    truncation=_SCREAMING_SNAKE_CASE ,    max_length=len(self.expected_src_tokens) ,    return_tensors="pt" ,    )
				__lowerCAmelCase   :						Dict				= shift_tokens_right(
				    batch["labels"] ,    self.tokenizer.pad_token_id ,    self.tokenizer.lang_code_to_id["ron_Latn"])

				self.assertIsInstance(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)

				self.assertEqual((2, 15) ,    batch.input_ids.shape)
				self.assertEqual((2, 15) ,    batch.attention_mask.shape)
				__lowerCAmelCase   :						List[Any]				= batch.input_ids.tolist()[0]
				self.assertListEqual(self.expected_src_tokens ,    _SCREAMING_SNAKE_CASE)
				self.assertEqual(_SCREAMING_SNAKE_CASE ,    batch.decoder_input_ids[0, 0])  # EOS
				# Test that special tokens are reset
				self.assertEqual(self.tokenizer.prefix_tokens ,    [EN_CODE])
				self.assertEqual(self.tokenizer.suffix_tokens ,    [self.tokenizer.eos_token_id])

	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= self.tokenizer(self.src_text ,    padding=_SCREAMING_SNAKE_CASE ,    truncation=_SCREAMING_SNAKE_CASE ,    max_length=3 ,    return_tensors="pt")
				__lowerCAmelCase   :						Optional[int]				= self.tokenizer(
				    text_target=self.tgt_text ,    padding=_SCREAMING_SNAKE_CASE ,    truncation=_SCREAMING_SNAKE_CASE ,    max_length=10 ,    return_tensors="pt")
				__lowerCAmelCase   :						Tuple				= targets["input_ids"]
				__lowerCAmelCase   :						Optional[int]				= shift_tokens_right(
				    _SCREAMING_SNAKE_CASE ,    self.tokenizer.pad_token_id ,    decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang] ,    )

				self.assertEqual(batch.input_ids.shape[1] ,    3)
				self.assertEqual(batch.decoder_input_ids.shape[1] ,    10)

	@require_torch
	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> List[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= self.tokenizer._build_translation_inputs(
				    "A test" ,    return_tensors="pt" ,    src_lang="eng_Latn" ,    tgt_lang="fra_Latn")

				self.assertEqual(
				    nested_simplify(_SCREAMING_SNAKE_CASE) ,    {
				        # A, test, EOS, en_XX
				        "input_ids": [[25_6047, 70, 7356, 2]],
				        "attention_mask": [[1, 1, 1, 1]],
				        # ar_AR
				        "forced_bos_token_id": 25_6057,
				    } ,    )







	@require_torch
	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						List[str]				= True
				__lowerCAmelCase   :						List[str]				= self.tokenizer(
				    "UN Chief says there is no military solution in Syria" ,    src_lang="eng_Latn" ,    tgt_lang="fra_Latn")
				self.assertEqual(
				    inputs.input_ids ,    [1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047])

				__lowerCAmelCase   :						Optional[Any]				= False
				__lowerCAmelCase   :						Any				= self.tokenizer(
				    "UN Chief says there is no military solution in Syria" ,    src_lang="eng_Latn" ,    tgt_lang="fra_Latn")
				self.assertEqual(
				    inputs.input_ids ,    [25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2])




"""simple docstring"""






import itertools
import random
import unittest

import numpy as np

from transformers import is_speech_available
from transformers.testing_utils import require_torch, require_torchaudio

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
	from transformers import SpeechaTextFeatureExtractor

__snake_case							:     Union[str, Any]   =					random.Random()


def     _lowercase       (						__snake_case      ,__snake_case=1.0      ,__snake_case=None      ,__snake_case=None			)		->     List[Any]:
			if rng is None:
						__lowerCAmelCase   :						Dict				= global_rng

			__lowerCAmelCase   :						Tuple				= []
			for batch_idx in range(shape[0]			):
						values.append([]			)
						for _ in range(shape[1]			):
									values[-1].append(rng.random() * scale			)

			return values



@require_torch
@require_torchaudio
class     A__     (						unittest.TestCase							):




	'''simple docstring'''





	def __init__(  self:	Optional[Any] ,    _SCREAMING_SNAKE_CASE:	Any ,    _SCREAMING_SNAKE_CASE:	List[Any]=7 ,    _SCREAMING_SNAKE_CASE:	int=400 ,    _SCREAMING_SNAKE_CASE:	List[str]=2000 ,    _SCREAMING_SNAKE_CASE:	Optional[Any]=24 ,    _SCREAMING_SNAKE_CASE:	Dict=24 ,    _SCREAMING_SNAKE_CASE:	Optional[int]=0.0 ,    _SCREAMING_SNAKE_CASE:	Any=1_6000 ,    _SCREAMING_SNAKE_CASE:	int=True ,    _SCREAMING_SNAKE_CASE:	Union[str, Any]=True ,    )       -> List[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						Any				= parent
				__lowerCAmelCase   :						str				= batch_size
				__lowerCAmelCase   :						List[str]				= min_seq_length
				__lowerCAmelCase   :						Any				= max_seq_length
				__lowerCAmelCase   :						Dict				= (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
				__lowerCAmelCase   :						Union[str, Any]				= feature_size
				__lowerCAmelCase   :						int				= num_mel_bins
				__lowerCAmelCase   :						Optional[Any]				= padding_value
				__lowerCAmelCase   :						List[Any]				= sampling_rate
				__lowerCAmelCase   :						Dict				= return_attention_mask
				__lowerCAmelCase   :						int				= do_normalize

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Optional[int]:

				"""simple docstring"""

				return {
				    "feature_size": self.feature_size,
				    "num_mel_bins": self.num_mel_bins,
				    "padding_value": self.padding_value,
				    "sampling_rate": self.sampling_rate,
				    "return_attention_mask": self.return_attention_mask,
				    "do_normalize": self.do_normalize,
				}







	def 							_SCREAMING_SNAKE_CASE     (  self:	Tuple ,    _SCREAMING_SNAKE_CASE:	Tuple=False ,    _SCREAMING_SNAKE_CASE:	int=False)       -> Optional[int]:

				"""simple docstring"""

				def _flatten(_SCREAMING_SNAKE_CASE:	Optional[Any]):
							return list(itertools.chain(*_SCREAMING_SNAKE_CASE))

				if equal_length:
							__lowerCAmelCase   :						Tuple				= [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
				else:
							# make sure that inputs increase in size
							__lowerCAmelCase   :						List[Any]				= [
							    floats_list((x, self.feature_size))
							    for x in range(self.min_seq_length ,    self.max_seq_length ,    self.seq_length_diff)
							]
				if numpify:
							__lowerCAmelCase   :						Any				= [np.asarray(_SCREAMING_SNAKE_CASE) for x in speech_inputs]
				return speech_inputs






@require_torch
@require_torchaudio
class     A__     (						__SCREAMING_SNAKE_CASE						,	unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE					   =			SpeechaTextFeatureExtractor if is_speech_available() else None

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> Tuple:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= SpeechaTextFeatureExtractionTester(self)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[Any] ,    _SCREAMING_SNAKE_CASE:	Optional[Any])       -> str:

				"""simple docstring"""

				self.assertTrue(np.all(np.mean(_SCREAMING_SNAKE_CASE ,    axis=0) < 1e-3))
				self.assertTrue(np.all(np.abs(np.var(_SCREAMING_SNAKE_CASE ,    axis=0) - 1) < 1e-3))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any])       -> int:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				# create three inputs of length 800, 1000, and 1200
				__lowerCAmelCase   :						Tuple				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						Optional[int]				= [np.asarray(_SCREAMING_SNAKE_CASE) for speech_input in speech_inputs]

				# Test feature size
				__lowerCAmelCase   :						Optional[int]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    padding=_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				self.assertTrue(input_features.ndim == 3)
				self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size)

				# Test not batched input
				__lowerCAmelCase   :						Optional[Any]				= feature_extractor(speech_inputs[0] ,    return_tensors="np").input_features
				__lowerCAmelCase   :						Any				= feature_extractor(np_speech_inputs[0] ,    return_tensors="np").input_features
				self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    atol=1e-3))

				# Test batched
				__lowerCAmelCase   :						Tuple				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				__lowerCAmelCase   :						Tuple				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				for enc_seq_a, enc_seq_a in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    atol=1e-3))

				# Test 2-D numpy arrays are batched.
				__lowerCAmelCase   :						List[str]				= [floats_list((1, x))[0] for x in (800, 800, 800)]
				__lowerCAmelCase   :						Dict				= np.asarray(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Optional[int]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				__lowerCAmelCase   :						Optional[Any]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				for enc_seq_a, enc_seq_a in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    atol=1e-3))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any])       -> List[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						Tuple				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]

				__lowerCAmelCase   :						Optional[Any]				= ["longest", "max_length", "do_not_pad"]
				__lowerCAmelCase   :						str				= [None, 16, None]
				for max_length, padding in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							__lowerCAmelCase   :						str				= feature_extractor(
							    _SCREAMING_SNAKE_CASE ,    padding=_SCREAMING_SNAKE_CASE ,    max_length=_SCREAMING_SNAKE_CASE ,    return_attention_mask=_SCREAMING_SNAKE_CASE)
							__lowerCAmelCase   :						Optional[Any]				= inputs.input_features
							__lowerCAmelCase   :						Optional[Any]				= inputs.attention_mask
							__lowerCAmelCase   :						Tuple				= [np.sum(_SCREAMING_SNAKE_CASE) for x in attention_mask]

							self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]])
							self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]])
							self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						Any				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]

				__lowerCAmelCase   :						Dict				= ["longest", "max_length", "do_not_pad"]
				__lowerCAmelCase   :						List[Any]				= [None, 16, None]
				for max_length, padding in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							__lowerCAmelCase   :						List[str]				= feature_extractor(
							    _SCREAMING_SNAKE_CASE ,    max_length=_SCREAMING_SNAKE_CASE ,    padding=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE)
							__lowerCAmelCase   :						List[str]				= inputs.input_features
							__lowerCAmelCase   :						Dict				= inputs.attention_mask
							__lowerCAmelCase   :						Dict				= [np.sum(_SCREAMING_SNAKE_CASE) for x in attention_mask]

							self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]])
							self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1e-6)
							self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]])
							self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1e-6)
							self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]])

	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> Any:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						str				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						str				= feature_extractor(
				    _SCREAMING_SNAKE_CASE ,    padding="max_length" ,    max_length=4 ,    truncation=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    )
				__lowerCAmelCase   :						List[str]				= inputs.input_features
				__lowerCAmelCase   :						Dict				= inputs.attention_mask
				__lowerCAmelCase   :						Any				= np.sum(attention_mask == 1 ,    axis=1)

				self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
				self._check_zero_mean_unit_variance(input_features[1])
				self._check_zero_mean_unit_variance(input_features[2])

	def 							_SCREAMING_SNAKE_CASE     (  self:	int)       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						List[Any]				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						Any				= feature_extractor(
				    _SCREAMING_SNAKE_CASE ,    padding="longest" ,    max_length=4 ,    truncation=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    )
				__lowerCAmelCase   :						List[str]				= inputs.input_features
				__lowerCAmelCase   :						Optional[Any]				= inputs.attention_mask
				__lowerCAmelCase   :						Optional[Any]				= np.sum(attention_mask == 1 ,    axis=1)

				self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
				self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]])
				self._check_zero_mean_unit_variance(input_features[2])

				# make sure that if max_length < longest -> then pad to max_length
				self.assertEqual(input_features.shape ,    (3, 4, 24))

				__lowerCAmelCase   :						List[str]				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						List[str]				= feature_extractor(
				    _SCREAMING_SNAKE_CASE ,    padding="longest" ,    max_length=16 ,    truncation=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    )
				__lowerCAmelCase   :						Optional[Any]				= inputs.input_features
				__lowerCAmelCase   :						Optional[int]				= inputs.attention_mask
				__lowerCAmelCase   :						str				= np.sum(attention_mask == 1 ,    axis=1)

				self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
				self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]])
				self._check_zero_mean_unit_variance(input_features[2])

				# make sure that if max_length < longest -> then pad to max_length
				self.assertEqual(input_features.shape ,    (3, 6, 24))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Dict:

				"""simple docstring"""

				import torch

				__lowerCAmelCase   :						Dict				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						List[str]				= np.random.rand(100 ,    32).astype(np.floataa)
				__lowerCAmelCase   :						List[Any]				= np_speech_inputs.tolist()

				for inputs in [py_speech_inputs, np_speech_inputs]:
							__lowerCAmelCase   :						Any				= feature_extractor.pad([{"input_features": inputs}] ,    return_tensors="np")
							self.assertTrue(np_processed.input_features.dtype == np.floataa)
							__lowerCAmelCase   :						List[Any]				= feature_extractor.pad([{"input_features": inputs}] ,    return_tensors="pt")
							self.assertTrue(pt_processed.input_features.dtype == torch.floataa)

	def 							_SCREAMING_SNAKE_CASE     (  self:	str ,    _SCREAMING_SNAKE_CASE:	List[Any])       -> Optional[Any]:

				"""simple docstring"""

				from datasets import load_dataset

				__lowerCAmelCase   :						Any				= load_dataset("hf-internal-testing/librispeech_asr_dummy" ,    "clean" ,    split="validation")
				# automatic decoding with librispeech
				__lowerCAmelCase   :						List[Any]				= ds.sort("id").select(range(_SCREAMING_SNAKE_CASE))[:num_samples]["audio"]

				return [x["array"] for x in speech_samples]







	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> Any:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= np.array([
				    -1.5745, -1.7713, -1.7020, -1.6069, -1.2250, -1.1105, -0.9072, -0.8241,
				    -1.2310, -0.8098, -0.3320, -0.4101, -0.7985, -0.4996, -0.8213, -0.9128,
				    -1.0420, -1.1286, -1.0440, -0.7999, -0.8405, -1.2275, -1.5443, -1.4625,
				])
				# fmt: on

				__lowerCAmelCase   :						str				= self._load_datasamples(1)
				__lowerCAmelCase   :						Any				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						List[str]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="pt").input_features
				self.assertEquals(input_features.shape ,    (1, 584, 24))
				self.assertTrue(np.allclose(input_features[0, 0, :30] ,    _SCREAMING_SNAKE_CASE ,    atol=1e-4))