



from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)



def    snake_case_				(    _SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   ):
	__lowercase		       =			[]
	for part_id in partition_order:
		__lowercase		       =			df.where(F"""SPARK_PARTITION_ID() = {part_id}"""   ).collect()
		for row_idx, row in enumerate(_SCREAMING_SNAKE_CASE   ):
			expected_row_ids_and_row_dicts.append((F"""{part_id}_{row_idx}""", row.asDict())   )
	return expected_row_ids_and_row_dicts



@require_not_windows
@require_dill_gt_0_3_2
def    snake_case_				(    ):
	__lowercase		       =			pyspark.sql.SparkSession.builder.master("local[*]"   ).appName("pyspark"   ).getOrCreate()
	__lowercase		       =			spark.range(1_0_0   ).repartition(1   )
	__lowercase		       =			Spark(_SCREAMING_SNAKE_CASE   )
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=1_6   )
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 5_0



@require_not_windows
@require_dill_gt_0_3_2
def    snake_case_				(    ):
	__lowercase		       =			pyspark.sql.SparkSession.builder.master("local[*]"   ).appName("pyspark"   ).getOrCreate()
	__lowercase		       =			spark.range(1_0   ).repartition(2   )
	__lowercase		       =			[1, 0]
	__lowercase		       =			_generate_iterable_examples(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   )  # Reverse the partitions.
	__lowercase		       =			_get_expected_row_ids_and_row_dicts_for_partition_order(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE   )

	for i, (row_id, row_dict) in enumerate(generate_fn()   ):
		__lowercase       ,    __lowercase		       =			expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict



@require_not_windows
@require_dill_gt_0_3_2
def    snake_case_				(    ):
	__lowercase		       =			pyspark.sql.SparkSession.builder.master("local[*]"   ).appName("pyspark"   ).getOrCreate()
	__lowercase		       =			spark.range(1_0   ).repartition(1   )
	__lowercase		       =			SparkExamplesIterable(_SCREAMING_SNAKE_CASE   )
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(_SCREAMING_SNAKE_CASE   ):
		assert row_id == F"""0_{i}"""
		assert row_dict == {"id": i}



@require_not_windows
@require_dill_gt_0_3_2
def    snake_case_				(    ):
	__lowercase		       =			pyspark.sql.SparkSession.builder.master("local[*]"   ).appName("pyspark"   ).getOrCreate()
	__lowercase		       =			spark.range(3_0   ).repartition(3   )
	# Mock the generator so that shuffle reverses the partition indices.
	with patch("numpy.random.Generator"   ) as generator_mock:
		__lowercase		       =			lambda _SCREAMING_SNAKE_CASE   : x.reverse()
		__lowercase		       =			_get_expected_row_ids_and_row_dicts_for_partition_order(_SCREAMING_SNAKE_CASE  ,	[2, 1, 0]   )

		__lowercase		       =			SparkExamplesIterable(_SCREAMING_SNAKE_CASE   ).shuffle_data_sources(_SCREAMING_SNAKE_CASE   )
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(_SCREAMING_SNAKE_CASE   ):
			__lowercase       ,    __lowercase		       =			expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict



@require_not_windows
@require_dill_gt_0_3_2
def    snake_case_				(    ):
	__lowercase		       =			pyspark.sql.SparkSession.builder.master("local[*]"   ).appName("pyspark"   ).getOrCreate()
	__lowercase		       =			spark.range(2_0   ).repartition(4   )

	# Partitions 0 and 2
	__lowercase		       =			SparkExamplesIterable(_SCREAMING_SNAKE_CASE   ).shard_data_sources(worker_id=0  ,	num_workers=2   )
	assert shard_it_a.n_shards == 2
	__lowercase		       =			_get_expected_row_ids_and_row_dicts_for_partition_order(_SCREAMING_SNAKE_CASE  ,	[0, 2]   )
	for i, (row_id, row_dict) in enumerate(_SCREAMING_SNAKE_CASE   ):
		__lowercase       ,    __lowercase		       =			expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	__lowercase		       =			SparkExamplesIterable(_SCREAMING_SNAKE_CASE   ).shard_data_sources(worker_id=1  ,	num_workers=2   )
	assert shard_it_a.n_shards == 2
	__lowercase		       =			_get_expected_row_ids_and_row_dicts_for_partition_order(_SCREAMING_SNAKE_CASE  ,	[1, 3]   )
	for i, (row_id, row_dict) in enumerate(_SCREAMING_SNAKE_CASE   ):
		__lowercase       ,    __lowercase		       =			expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict



@require_not_windows
@require_dill_gt_0_3_2
def    snake_case_				(    ):
	__lowercase		       =			pyspark.sql.SparkSession.builder.master("local[*]"   ).appName("pyspark"   ).getOrCreate()
	__lowercase		       =			spark.range(1_0_0   ).repartition(1   )
	__lowercase		       =			Spark(_SCREAMING_SNAKE_CASE   )
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1   )
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 1_0_0








# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.whisper import WhisperForConditionalGeneration, WhisperProcessor
from .base import PipelineTool

class   _A   ( _lowercase	):


						'''simple docstring'''





						_snake_case			:     Dict      							=      """openai/whisper-base"""
						_snake_case			:     Union[str, Any]      							=      (
						    """This is a tool that transcribes an audio into text. It takes an input named `audio` and returns the """
						    """transcribed text."""
						)
						_snake_case			:     Any      							=      """transcriber"""
						_snake_case			:     Any      							=      WhisperProcessor
						_snake_case			:     Optional[int]      							=      WhisperForConditionalGeneration

						_snake_case			:     str      							=      ["""audio"""]
						_snake_case			:     Optional[int]      							=      ["""text"""]



						def     _snake_case	(	self	:							List[str]							,      lowerCamelCase	:							Optional[int]	):






							'''simple docstring'''



							return self.pre_processor(lowerCamelCase							,      return_tensors="pt"	).input_features



						def     _snake_case	(	self	:							str							,      lowerCamelCase	:							List[Any]	):






							'''simple docstring'''



							return self.model.generate(inputs=lowerCamelCase	)



						def     _snake_case	(	self	:							List[str]							,      lowerCamelCase	:							Optional[Any]	):






							'''simple docstring'''



							return self.pre_processor.batch_decode(lowerCamelCase							,      skip_special_tokens=lowerCamelCase	)[0]



