


import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

_UpperCAmelCase			:  int     =logging.get_logger(__name__)


_UpperCAmelCase			:  Tuple     ={name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}



def 			lowerCAmelCase (		lowerCAmelCase_ ,   lowerCAmelCase_ ,   lowerCAmelCase_ ,   lowerCAmelCase_ )-> Dict:
				if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
								raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys() )}.""" )

				if tokenizer_name is None:
								lowerCAmelCase_ :      Tuple    				=				TOKENIZER_CLASSES
				else:
								lowerCAmelCase_ :      Union[str, Any]    				=				{tokenizer_name: getattr(lowerCAmelCase_ ,   tokenizer_name + '''Fast''' )}

				logger.info(f"""Loading tokenizer classes: {tokenizer_names}""" )

				for tokenizer_name in tokenizer_names:
								lowerCAmelCase_ :      Optional[Any]    				=				TOKENIZER_CLASSES[tokenizer_name]

								lowerCAmelCase_ :      List[str]    				=				True
								if checkpoint_name is None:
												lowerCAmelCase_ :      Tuple    				=				list(tokenizer_class.max_model_input_sizes.keys() )
								else:
												lowerCAmelCase_ :      Union[str, Any]    				=				[checkpoint_name]

								logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}""" )

								for checkpoint in checkpoint_names:
												logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}""" )

												# Load tokenizer
												lowerCAmelCase_ :      str    				=				tokenizer_class.from_pretrained(lowerCAmelCase_ ,   force_download=lowerCAmelCase_ )

												# Save fast tokenizer
												logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}""" )

												# For organization names we create sub-directories
												if "/" in checkpoint:
																lowerCAmelCase_ :      Union[str, Any]    				=				checkpoint.split('''/''' )
																lowerCAmelCase_ :      Dict    				=				os.path.join(lowerCAmelCase_ ,   lowerCAmelCase_ )
												elif add_prefix:
																lowerCAmelCase_ :      int    				=				checkpoint
																lowerCAmelCase_ :      Any    				=				dump_path
												else:
																lowerCAmelCase_ :      Any    				=				None
																lowerCAmelCase_ :      Any    				=				dump_path

												logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}""" )

												if checkpoint in list(tokenizer.pretrained_vocab_files_map.values() )[0]:
																lowerCAmelCase_ :      List[str]    				=				list(tokenizer.pretrained_vocab_files_map.values() )[0][checkpoint]
																lowerCAmelCase_ :      List[str]    				=				file_path.split(lowerCAmelCase_ )[-1][0]
																if next_char == "/":
																				lowerCAmelCase_ :      Dict    				=				os.path.join(lowerCAmelCase_ ,   lowerCAmelCase_ )
																				lowerCAmelCase_ :      Union[str, Any]    				=				None

																logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}""" )

												lowerCAmelCase_ :      List[Any]    				=				tokenizer.save_pretrained(
												    lowerCAmelCase_ ,   legacy_format=lowerCAmelCase_ ,   filename_prefix=lowerCAmelCase_ )
												logger.info(f"""=> File names {file_names}""" )

												for file_name in file_names:
																if not file_name.endswith('''tokenizer.json''' ):
																				os.remove(lowerCAmelCase_ )
																				logger.info(f"""=> removing {file_name}""" )


if __name__ == "__main__":
						_UpperCAmelCase			:  Tuple     =argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
						)
						parser.add_argument(
						    """--tokenizer_name""",
						    default=None,
						    type=str,
						    help=(
						        f"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
						        """download and convert all the checkpoints from AWS."""
						    ),
						)
						parser.add_argument(
						    """--checkpoint_name""",
						    default=None,
						    type=str,
						    help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
						)
						parser.add_argument(
						    """--force_download""",
						    action="""store_true""",
						    help="""Re-download checkpoints.""",
						)
						_UpperCAmelCase			:  Optional[int]     =parser.parse_args()

						convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)










import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Audio, Features, Value
from .base import TaskTemplate







@dataclass(frozen=UpperCAmelCase__							)
class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  str					=							field(default="""automatic-speech-recognition""",						metadata={"""include_in_asdict_even_if_is_default""": True}							)
						SCREAMING_SNAKE_CASE__      :  ClassVar[Features]					=							Features({"""audio""": Audio()}							)
						SCREAMING_SNAKE_CASE__      :  ClassVar[Features]					=							Features({"""transcription""": Value("""string"""							)}							)
						SCREAMING_SNAKE_CASE__      :  str					=							"audio"
						SCREAMING_SNAKE_CASE__      :  str					=							"transcription"




						def 						lowercase_ (				self   ,						__lowercase							)					->    int:
										if self.audio_column not in features:
														raise ValueError(f"""Column {self.audio_column} is not present in features."""							)
										if not isinstance(features[self.audio_column]   ,						__lowercase							):
														raise ValueError(f"""Column {self.audio_column} is not an Audio type."""							)
										lowerCAmelCase_ :      List[str]    				=				copy.deepcopy(self							)
										lowerCAmelCase_ :      Optional[Any]    				=				self.input_schema.copy()
										lowerCAmelCase_ :      Optional[Any]    				=				features[self.audio_column]
										lowerCAmelCase_ :      List[str]    				=				input_schema
										return task_template




						@property
						def 						lowercase_ (				self							)					->    Dict[str, str]:
										return {self.audio_column: "audio", self.transcription_column: "transcription"}