





import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
lowerCAmelCase_						   =    """src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
lowerCAmelCase_						   =    direct_transformers_import(PATH_TO_TRANSFORMERS)

lowerCAmelCase_						   =    transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
lowerCAmelCase_						   =    re.compile(r"""\[(.+?)\]\((https://huggingface\.co/.+?)\)""")


lowerCAmelCase_						   =    {
    """DecisionTransformerConfig""",
    """EncoderDecoderConfig""",
    """MusicgenConfig""",
    """RagConfig""",
    """SpeechEncoderDecoderConfig""",
    """TimmBackboneConfig""",
    """VisionEncoderDecoderConfig""",
    """VisionTextDualEncoderConfig""",
    """LlamaConfig""",
}


def     lowerCamelCase_	(      lowerCAmelCase:	int				)->		List[Any]:
  _snake_case   :				Dict														=	None

  # source code of `config_class`
  _snake_case   :				List[Any]														=	inspect.getsource(lowerCAmelCase				)
  _snake_case   :				List[Any]														=	_re_checkpoint.findall(lowerCAmelCase				)

  # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
  # For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
  for ckpt_name, ckpt_link in checkpoints:
    # allow the link to end with `/`
    if ckpt_link.endswith('/'				):
      _snake_case   :				Optional[int]														=	ckpt_link[:-1]

    # verify the checkpoint name corresponds to the checkpoint link
    _snake_case   :				Dict														=	F"""https://huggingface.co/{ckpt_name}"""
    if ckpt_link == ckpt_link_from_name:
      _snake_case   :				Any														=	ckpt_name
      break

  return checkpoint


def     lowerCamelCase_	(      )->		int:
  _snake_case   :				List[Any]														=	[]

  for config_class in list(CONFIG_MAPPING.values()				):
    # Skip deprecated models
    if "models.deprecated" in config_class.__module__:
      continue
    _snake_case   :				List[Any]														=	get_checkpoint_from_config_class(lowerCAmelCase				)

    _snake_case   :				int														=	config_class.__name__
    if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
      configs_without_checkpoint.append(lowerCAmelCase				)

  if len(lowerCAmelCase				) > 0:
    _snake_case   :				List[str]														=	'\n'.join(sorted(lowerCAmelCase				)				)
    raise ValueError(F"""The following configurations don't contain any valid checkpoint:\n{message}"""				)


if __name__ == "__main__":
      check_config_docstrings_have_checkpoints()











import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCAmelCase_						   =    logging.get_logger(__name__)


lowerCAmelCase_						   =    {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}


def     lowerCamelCase_	(      lowerCAmelCase:	List[Any]    ,							lowerCAmelCase:	Optional[Any]    ,							lowerCAmelCase:	Dict    ,							lowerCAmelCase:	Union[str, Any]				)->		Optional[int]:
  if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
    raise ValueError(F"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()				)}."""				)

  if tokenizer_name is None:
    _snake_case   :				Tuple														=	TOKENIZER_CLASSES
  else:
    _snake_case   :				Union[str, Any]														=	{tokenizer_name: getattr(lowerCAmelCase    ,							tokenizer_name + 'Fast'				)}

  logger.info(F"""Loading tokenizer classes: {tokenizer_names}"""				)

  for tokenizer_name in tokenizer_names:
    _snake_case   :				Dict														=	TOKENIZER_CLASSES[tokenizer_name]

    _snake_case   :				Optional[Any]														=	True
    if checkpoint_name is None:
      _snake_case   :				Union[str, Any]														=	list(tokenizer_class.max_model_input_sizes.keys()				)
    else:
      _snake_case   :				Optional[int]														=	[checkpoint_name]

    logger.info(F"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""				)

    for checkpoint in checkpoint_names:
      logger.info(F"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""				)

      # Load tokenizer
      _snake_case   :				str														=	tokenizer_class.from_pretrained(lowerCAmelCase    ,							force_download=lowerCAmelCase				)

      # Save fast tokenizer
      logger.info(F"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""				)

      # For organization names we create sub-directories
      if "/" in checkpoint:
        _snake_case							,					_snake_case   :				Tuple														=	checkpoint.split('/'				)
        _snake_case   :				int														=	os.path.join(lowerCAmelCase    ,							lowerCAmelCase				)
      elif add_prefix:
        _snake_case   :				Dict														=	checkpoint
        _snake_case   :				Optional[Any]														=	dump_path
      else:
        _snake_case   :				str														=	None
        _snake_case   :				Union[str, Any]														=	dump_path

      logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""				)

      if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()				)[0]:
        _snake_case   :				Optional[Any]														=	list(tokenizer.pretrained_vocab_files_map.values()				)[0][checkpoint]
        _snake_case   :				Optional[int]														=	file_path.split(lowerCAmelCase				)[-1][0]
        if next_char == "/":
          _snake_case   :				Union[str, Any]														=	os.path.join(lowerCAmelCase    ,							lowerCAmelCase				)
          _snake_case   :				str														=	None

        logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""				)

      _snake_case   :				Optional[int]														=	tokenizer.save_pretrained(
          lowerCAmelCase    ,							legacy_format=lowerCAmelCase    ,							filename_prefix=lowerCAmelCase				)
      logger.info(F"""=> File names {file_names}"""				)

      for file_name in file_names:
        if not file_name.endswith('tokenizer.json'				):
          os.remove(lowerCAmelCase				)
          logger.info(F"""=> removing {file_name}"""				)


if __name__ == "__main__":
      lowerCAmelCase_						   =    argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
      )
      parser.add_argument(
          """--tokenizer_name""",
          default=None,
          type=str,
          help=(
              F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
              """download and convert all the checkpoints from AWS."""
          ),
      )
      parser.add_argument(
          """--checkpoint_name""",
          default=None,
          type=str,
          help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
      )
      parser.add_argument(
          """--force_download""",
          action="""store_true""",
          help="""Re-download checkpoints.""",
      )
      lowerCAmelCase_						   =    parser.parse_args()

      convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)




