






'''simple docstring'''







from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
		from ...feature_extraction_utils import FeatureExtractionMixin
		from ...tokenization_utils_base import PreTrainedTokenizerBase
		from ...utils import TensorType

__lowerCAmelCase       		=      logging.get_logger(__name__)

__lowerCAmelCase       		=      {
    'openai/whisper-base': 'https://huggingface.co/openai/whisper-base/resolve/main/config.json',
}

# fmt: off
__lowerCAmelCase       		=      [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1_058, 1_220, 1_267, 1_279, 1_303, 1_343, 1_377,
    1_391, 1_635, 1_782, 1_875, 2_162, 2_361, 2_488, 3_467, 4_008, 4_211,
    4_600, 4_808, 5_299, 5_855, 6_329, 7_203, 9_609, 9_959, 10_563, 10_786,
    11_420, 11_709, 11_907, 13_163, 13_697, 13_700, 14_808, 15_306, 16_410, 16_791,
    17_992, 19_203, 19_510, 20_724, 22_305, 22_935, 27_007, 30_109, 30_420, 33_409,
    34_949, 40_283, 40_493, 40_549, 47_282, 49_146, 50_257, 50_359, 50_360, 50_361
]
__lowerCAmelCase       		=      [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1_350, 1_853, 1_982, 2_460, 2_627,
    3_246, 3_253, 3_268, 3_536, 3_846, 3_961, 4_183, 4_667, 6_585, 6_647,
    7_273, 9_061, 9_383, 10_428, 10_929, 11_938, 12_033, 12_331, 12_562, 13_793,
    14_157, 14_635, 15_265, 15_618, 16_553, 16_604, 18_362, 18_956, 20_075, 21_675,
    22_520, 26_130, 26_161, 26_435, 28_279, 29_464, 31_650, 32_302, 32_470, 36_865,
    42_863, 47_425, 49_870, 50_254, 50_258, 50_360, 50_361, 50_362
]





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):
			snake_case__  =				"whisper"
			snake_case__  =				["past_key_values"]
			snake_case__  =				{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}






			def __init__( self      :			Union[str, Any]  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=5_1865  ,						__SCREAMING_SNAKE_CASE      :			List[str]=80  ,						__SCREAMING_SNAKE_CASE      :			Dict=6  ,						__SCREAMING_SNAKE_CASE      :			Union[str, Any]=4  ,						__SCREAMING_SNAKE_CASE      :			Any=6  ,						__SCREAMING_SNAKE_CASE      :			Any=4  ,						__SCREAMING_SNAKE_CASE      :			Dict=1536  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]=1536  ,						__SCREAMING_SNAKE_CASE      :			int=0.0  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=0.0  ,						__SCREAMING_SNAKE_CASE      :			Tuple=5_0257  ,						__SCREAMING_SNAKE_CASE      :			List[str]=True  ,						__SCREAMING_SNAKE_CASE      :			str=True  ,						__SCREAMING_SNAKE_CASE      :			int="gelu"  ,						__SCREAMING_SNAKE_CASE      :			Any=256  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=0.0  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]=0.0  ,						__SCREAMING_SNAKE_CASE      :			Dict=0.0  ,						__SCREAMING_SNAKE_CASE      :			int=0.02  ,						__SCREAMING_SNAKE_CASE      :			Dict=False  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=1500  ,						__SCREAMING_SNAKE_CASE      :			Any=448  ,						__SCREAMING_SNAKE_CASE      :			str=5_0256  ,						__SCREAMING_SNAKE_CASE      :			Optional[int]=5_0256  ,						__SCREAMING_SNAKE_CASE      :			Tuple=5_0256  ,						__SCREAMING_SNAKE_CASE      :			Any=None  ,						__SCREAMING_SNAKE_CASE      :			List[Any]=[220, 5_0256]  ,						__SCREAMING_SNAKE_CASE      :			Union[str, Any]=False  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=256  ,						__SCREAMING_SNAKE_CASE      :			Optional[Any]=False  ,						__SCREAMING_SNAKE_CASE      :			Dict=0.05  ,						__SCREAMING_SNAKE_CASE      :			Any=10  ,						__SCREAMING_SNAKE_CASE      :			Union[str, Any]=2  ,						__SCREAMING_SNAKE_CASE      :			Any=0.0  ,						__SCREAMING_SNAKE_CASE      :			int=10  ,						__SCREAMING_SNAKE_CASE      :			Tuple=0  ,						__SCREAMING_SNAKE_CASE      :			str=7  ,						**__SCREAMING_SNAKE_CASE      :			str  ,						)			->			List[str]:
							a_		:  Optional[int]											=      vocab_size
							a_		:  Dict											=      num_mel_bins
							a_		:  Union[str, Any]											=      d_model
							a_		:  str											=      encoder_layers
							a_		:  Union[str, Any]											=      encoder_attention_heads
							a_		:  str											=      decoder_layers
							a_		:  Optional[Any]											=      decoder_attention_heads
							a_		:  List[Any]											=      decoder_ffn_dim
							a_		:  Optional[Any]											=      encoder_ffn_dim
							a_		:  int											=      dropout
							a_		:  str											=      attention_dropout
							a_		:  Tuple											=      activation_dropout
							a_		:  List[str]											=      activation_function
							a_		:  int											=      init_std
							a_		:  str											=      encoder_layerdrop
							a_		:  int											=      decoder_layerdrop
							a_		:  Union[str, Any]											=      use_cache
							a_		:  Any											=      encoder_layers
							a_		:  Optional[int]											=      scale_embedding  # scale factor will be sqrt(d_model) if True
							a_		:  Optional[int]											=      max_source_positions
							a_		:  Optional[int]											=      max_target_positions

							# Audio Classification-specific parameters. Feel free to ignore for other classes.
							a_		:  Optional[Any]											=      classifier_proj_size
							a_		:  int											=      use_weighted_layer_sum

							# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
							a_		:  str											=      apply_spec_augment
							a_		:  List[Any]											=      mask_time_prob
							a_		:  int											=      mask_time_length
							a_		:  Tuple											=      mask_time_min_masks
							a_		:  Any											=      mask_feature_prob
							a_		:  Optional[Any]											=      mask_feature_length
							a_		:  Tuple											=      mask_feature_min_masks

							a_		:  List[Any]											=      median_filter_width

							super().__init__(
							    pad_token_id=__SCREAMING_SNAKE_CASE  ,						bos_token_id=__SCREAMING_SNAKE_CASE  ,						eos_token_id=__SCREAMING_SNAKE_CASE  ,						is_encoder_decoder=__SCREAMING_SNAKE_CASE  ,						decoder_start_token_id=__SCREAMING_SNAKE_CASE  ,						suppress_tokens=__SCREAMING_SNAKE_CASE  ,						begin_suppress_tokens=__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE  ,						)





class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_       ):






			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict					)			->			Mapping[str, Mapping[int, str]]:
							a_		:  Optional[Any]											=      OrderedDict(
							    [
							        ('''input_features''', {0: '''batch''', 1: '''feature_size''', 2: '''encoder_sequence'''}),
							    ]					)
							if self.use_past:
											a_		:  Tuple											=      {0: '''batch'''}
							else:
											a_		:  Tuple											=      {0: '''batch''', 1: '''decoder_sequence'''}

							if self.use_past:
											self.fill_with_past_key_values_(__SCREAMING_SNAKE_CASE  ,						direction='''inputs'''					)

							return common_inputs






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[Any]  ,						__SCREAMING_SNAKE_CASE      :			Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]  ,						__SCREAMING_SNAKE_CASE      :			int = -1  ,						__SCREAMING_SNAKE_CASE      :			int = -1  ,						__SCREAMING_SNAKE_CASE      :			bool = False  ,						__SCREAMING_SNAKE_CASE      :			Optional["TensorType"] = None  ,						__SCREAMING_SNAKE_CASE      :			int = 2_2050  ,						__SCREAMING_SNAKE_CASE      :			float = 5.0  ,						__SCREAMING_SNAKE_CASE      :			int = 220  ,						)			->			Mapping[str, Any]:
							a_		:  List[str]											=      OrderedDict()
							a_		:  int											=      OnnxConfig.generate_dummy_inputs(
							    self  ,						preprocessor=preprocessor.feature_extractor  ,						batch_size=__SCREAMING_SNAKE_CASE  ,						framework=__SCREAMING_SNAKE_CASE  ,						sampling_rate=__SCREAMING_SNAKE_CASE  ,						time_duration=__SCREAMING_SNAKE_CASE  ,						frequency=__SCREAMING_SNAKE_CASE  ,						)
							a_		:  Optional[Any]											=      encoder_inputs['''input_features'''].shape[2]
							a_		:  int											=      encoder_sequence_length // 2 if self.use_past else seq_length

							a_		:  Optional[Any]											=      super().generate_dummy_inputs(
							    preprocessor.tokenizer  ,						__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							a_		:  Union[str, Any]											=      encoder_inputs.pop('''input_features'''					)
							a_		:  Dict											=      decoder_inputs.pop('''decoder_input_ids'''					)

							if "past_key_values" in decoder_inputs:
											a_		:  Tuple											=      decoder_inputs.pop('''past_key_values'''					)

							return dummy_inputs







			@property
			def 			SCREAMING_SNAKE_CASE      ( self      :			str					)			->			float:
							return 1e-3












'''simple docstring'''







from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)





def 		_UpperCAmelCase   (							__A							:   List[str]						,		__A							:   List[Any]      ):
				a_		:  Any											=      []
				for part_id in partition_order:
								a_		:  str											=      df.where(f'SPARK_PARTITION_ID() = {part_id}'      ).collect()
								for row_idx, row in enumerate(__A      ):
												expected_row_ids_and_row_dicts.append((f'{part_id}_{row_idx}', row.asDict())      )
				return expected_row_ids_and_row_dicts





@require_not_windows
@require_dill_gt_0_3_2
def 		_UpperCAmelCase   (							):
				a_		:  List[str]											=      pyspark.sql.SparkSession.builder.master('''local[*]'''      ).appName('''pyspark'''      ).getOrCreate()
				a_		:  Union[str, Any]											=      spark.range(1_00      ).repartition(1      )
				a_		:  Any											=      Spark(__A      )
				# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
				# that each partition can hold 2 rows.
				spark_builder._repartition_df_if_needed(max_shard_size=16      )
				# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
				assert spark_builder.df.rdd.getNumPartitions() == 50





@require_not_windows
@require_dill_gt_0_3_2
def 		_UpperCAmelCase   (							):
				a_		:  List[Any]											=      pyspark.sql.SparkSession.builder.master('''local[*]'''      ).appName('''pyspark'''      ).getOrCreate()
				a_		:  int											=      spark.range(10      ).repartition(2      )
				a_		:  Tuple											=      [1, 0]
				a_		:  List[str]											=      _generate_iterable_examples(__A						,		__A      )  # Reverse the partitions.
				a_		:  int											=      _get_expected_row_ids_and_row_dicts_for_partition_order(__A						,		__A      )

				for i, (row_id, row_dict) in enumerate(generate_fn()      ):
								a_				,			a_		:  List[Any]											=      expected_row_ids_and_row_dicts[i]
								assert row_id == expected_row_id
								assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def 		_UpperCAmelCase   (							):
				a_		:  int											=      pyspark.sql.SparkSession.builder.master('''local[*]'''      ).appName('''pyspark'''      ).getOrCreate()
				a_		:  str											=      spark.range(10      ).repartition(1      )
				a_		:  Tuple											=      SparkExamplesIterable(__A      )
				assert it.n_shards == 1
				for i, (row_id, row_dict) in enumerate(__A      ):
								assert row_id == f'0_{i}'
								assert row_dict == {"id": i}





@require_not_windows
@require_dill_gt_0_3_2
def 		_UpperCAmelCase   (							):
				a_		:  Tuple											=      pyspark.sql.SparkSession.builder.master('''local[*]'''      ).appName('''pyspark'''      ).getOrCreate()
				a_		:  str											=      spark.range(30      ).repartition(3      )
				# Mock the generator so that shuffle reverses the partition indices.
				with patch('''numpy.random.Generator'''      ) as generator_mock:
								a_		:  Union[str, Any]											=      lambda __A      : x.reverse()
								a_		:  Any											=      _get_expected_row_ids_and_row_dicts_for_partition_order(__A						,		[2, 1, 0]      )

								a_		:  str											=      SparkExamplesIterable(__A      ).shuffle_data_sources(__A      )
								assert shuffled_it.n_shards == 3
								for i, (row_id, row_dict) in enumerate(__A      ):
												a_				,			a_		:  Optional[Any]											=      expected_row_ids_and_row_dicts[i]
												assert row_id == expected_row_id
												assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def 		_UpperCAmelCase   (							):
				a_		:  int											=      pyspark.sql.SparkSession.builder.master('''local[*]'''      ).appName('''pyspark'''      ).getOrCreate()
				a_		:  List[str]											=      spark.range(20      ).repartition(4      )

				# Partitions 0 and 2
				a_		:  Dict											=      SparkExamplesIterable(__A      ).shard_data_sources(worker_id=0						,		num_workers=2      )
				assert shard_it_a.n_shards == 2
				a_		:  Optional[Any]											=      _get_expected_row_ids_and_row_dicts_for_partition_order(__A						,		[0, 2]      )
				for i, (row_id, row_dict) in enumerate(__A      ):
								a_				,			a_		:  Tuple											=      expected_row_ids_and_row_dicts_a[i]
								assert row_id == expected_row_id
								assert row_dict == expected_row_dict

				# Partitions 1 and 3
				a_		:  List[Any]											=      SparkExamplesIterable(__A      ).shard_data_sources(worker_id=1						,		num_workers=2      )
				assert shard_it_a.n_shards == 2
				a_		:  Optional[int]											=      _get_expected_row_ids_and_row_dicts_for_partition_order(__A						,		[1, 3]      )
				for i, (row_id, row_dict) in enumerate(__A      ):
								a_				,			a_		:  Any											=      expected_row_ids_and_row_dicts_a[i]
								assert row_id == expected_row_id
								assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def 		_UpperCAmelCase   (							):
				a_		:  Any											=      pyspark.sql.SparkSession.builder.master('''local[*]'''      ).appName('''pyspark'''      ).getOrCreate()
				a_		:  List[Any]											=      spark.range(1_00      ).repartition(1      )
				a_		:  Optional[Any]											=      Spark(__A      )
				# Choose a small max_shard_size for maximum partitioning.
				spark_builder._repartition_df_if_needed(max_shard_size=1      )
				# The new number of partitions should not be greater than the number of rows.
				assert spark_builder.df.rdd.getNumPartitions() == 1_00




