





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase_						   =    logging.get_logger(__name__)

lowerCAmelCase_						   =    {
    """google/pegasus-large""": """https://huggingface.co/google/pegasus-large/resolve/main/config.json""",
    # See all PEGASUS models at https://huggingface.co/models?filter=pegasus
}



class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       Any      					="""pegasus"""
  a_							:       List[Any]      					=["""past_key_values"""]
  a_							:       Any      					={"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}







  def __init__(      self      :					Tuple      ,				UpperCamelCase      :					Optional[int]=5_02_65      ,				UpperCamelCase      :					Optional[int]=10_24      ,				UpperCamelCase      :					Any=12      ,				UpperCamelCase      :					Tuple=40_96      ,				UpperCamelCase      :					Tuple=16      ,				UpperCamelCase      :					int=12      ,				UpperCamelCase      :					Optional[int]=40_96      ,				UpperCamelCase      :					Tuple=16      ,				UpperCamelCase      :					str=0.0      ,				UpperCamelCase      :					Union[str, Any]=0.0      ,				UpperCamelCase      :					List[Any]=True      ,				UpperCamelCase      :					Optional[int]=True      ,				UpperCamelCase      :					Dict="gelu"      ,				UpperCamelCase      :					Optional[int]=10_24      ,				UpperCamelCase      :					Optional[int]=0.1      ,				UpperCamelCase      :					str=0.0      ,				UpperCamelCase      :					Union[str, Any]=0.0      ,				UpperCamelCase      :					Dict=0.02      ,				UpperCamelCase      :					str=0      ,				UpperCamelCase      :					Any=False      ,				UpperCamelCase      :					List[str]=0      ,				UpperCamelCase      :					Optional[int]=1      ,				UpperCamelCase      :					int=1      ,				**UpperCamelCase      :					str      ,				):


    '''simple docstring'''





    _snake_case   :				int														=	vocab_size
    _snake_case   :				Optional[int]														=	max_position_embeddings
    _snake_case   :				Dict														=	d_model
    _snake_case   :				List[str]														=	encoder_ffn_dim
    _snake_case   :				int														=	encoder_layers
    _snake_case   :				Optional[Any]														=	encoder_attention_heads
    _snake_case   :				Optional[Any]														=	decoder_ffn_dim
    _snake_case   :				Optional[Any]														=	decoder_layers
    _snake_case   :				List[str]														=	decoder_attention_heads
    _snake_case   :				str														=	dropout
    _snake_case   :				Union[str, Any]														=	attention_dropout
    _snake_case   :				List[str]														=	activation_dropout
    _snake_case   :				Optional[Any]														=	activation_function
    _snake_case   :				Dict														=	init_std
    _snake_case   :				str														=	encoder_layerdrop
    _snake_case   :				Optional[int]														=	decoder_layerdrop
    _snake_case   :				Optional[int]														=	use_cache
    _snake_case   :				Union[str, Any]														=	encoder_layers
    _snake_case   :				List[Any]														=	scale_embedding  # scale factor will be sqrt(d_model) if True
    super().__init__(
        pad_token_id=UpperCamelCase      ,				eos_token_id=UpperCamelCase      ,				is_encoder_decoder=UpperCamelCase      ,				decoder_start_token_id=UpperCamelCase      ,				forced_eos_token_id=UpperCamelCase      ,				**UpperCamelCase      ,				)







  @property
  def UpperCamelCase_ (      self      :					Tuple							):


    '''simple docstring'''





    return self.encoder_attention_heads







  @property
  def UpperCamelCase_ (      self      :					int							):


    '''simple docstring'''





    return self.d_model











import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCAmelCase_						   =    logging.get_logger(__name__)


lowerCAmelCase_						   =    {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}


def     lowerCamelCase_	(      lowerCAmelCase:	List[Any]    ,							lowerCAmelCase:	Optional[Any]    ,							lowerCAmelCase:	Dict    ,							lowerCAmelCase:	Union[str, Any]				)->		Optional[int]:
  if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
    raise ValueError(F"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()				)}."""				)

  if tokenizer_name is None:
    _snake_case   :				Tuple														=	TOKENIZER_CLASSES
  else:
    _snake_case   :				Union[str, Any]														=	{tokenizer_name: getattr(lowerCAmelCase    ,							tokenizer_name + 'Fast'				)}

  logger.info(F"""Loading tokenizer classes: {tokenizer_names}"""				)

  for tokenizer_name in tokenizer_names:
    _snake_case   :				Dict														=	TOKENIZER_CLASSES[tokenizer_name]

    _snake_case   :				Optional[Any]														=	True
    if checkpoint_name is None:
      _snake_case   :				Union[str, Any]														=	list(tokenizer_class.max_model_input_sizes.keys()				)
    else:
      _snake_case   :				Optional[int]														=	[checkpoint_name]

    logger.info(F"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""				)

    for checkpoint in checkpoint_names:
      logger.info(F"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""				)

      # Load tokenizer
      _snake_case   :				str														=	tokenizer_class.from_pretrained(lowerCAmelCase    ,							force_download=lowerCAmelCase				)

      # Save fast tokenizer
      logger.info(F"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""				)

      # For organization names we create sub-directories
      if "/" in checkpoint:
        _snake_case							,					_snake_case   :				Tuple														=	checkpoint.split('/'				)
        _snake_case   :				int														=	os.path.join(lowerCAmelCase    ,							lowerCAmelCase				)
      elif add_prefix:
        _snake_case   :				Dict														=	checkpoint
        _snake_case   :				Optional[Any]														=	dump_path
      else:
        _snake_case   :				str														=	None
        _snake_case   :				Union[str, Any]														=	dump_path

      logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""				)

      if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()				)[0]:
        _snake_case   :				Optional[Any]														=	list(tokenizer.pretrained_vocab_files_map.values()				)[0][checkpoint]
        _snake_case   :				Optional[int]														=	file_path.split(lowerCAmelCase				)[-1][0]
        if next_char == "/":
          _snake_case   :				Union[str, Any]														=	os.path.join(lowerCAmelCase    ,							lowerCAmelCase				)
          _snake_case   :				str														=	None

        logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""				)

      _snake_case   :				Optional[int]														=	tokenizer.save_pretrained(
          lowerCAmelCase    ,							legacy_format=lowerCAmelCase    ,							filename_prefix=lowerCAmelCase				)
      logger.info(F"""=> File names {file_names}"""				)

      for file_name in file_names:
        if not file_name.endswith('tokenizer.json'				):
          os.remove(lowerCAmelCase				)
          logger.info(F"""=> removing {file_name}"""				)


if __name__ == "__main__":
      lowerCAmelCase_						   =    argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
      )
      parser.add_argument(
          """--tokenizer_name""",
          default=None,
          type=str,
          help=(
              F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
              """download and convert all the checkpoints from AWS."""
          ),
      )
      parser.add_argument(
          """--checkpoint_name""",
          default=None,
          type=str,
          help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
      )
      parser.add_argument(
          """--force_download""",
          action="""store_true""",
          help="""Re-download checkpoints.""",
      )
      lowerCAmelCase_						   =    parser.parse_args()

      convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)




