
"""simple docstring"""




import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCAmelCase__							=				logging.get_logger(__name__)


lowerCAmelCase__							=				{name: getattr(transformers, name + 'Fast') for name in SLOW_TO_FAST_CONVERTERS}



def        _lowerCamelCase ( __a,							__a,							__a,							__a    ):
		if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
				raise ValueError(F'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()    )}.'    )

		if tokenizer_name is None:
				SCREAMING_SNAKE_CASE_						 = TOKENIZER_CLASSES
		else:
				SCREAMING_SNAKE_CASE_						 = {tokenizer_name: getattr(__a,							tokenizer_name + '''Fast'''    )}

		logger.info(F'Loading tokenizer classes: {tokenizer_names}'    )

		for tokenizer_name in tokenizer_names:
				SCREAMING_SNAKE_CASE_						 = TOKENIZER_CLASSES[tokenizer_name]

				SCREAMING_SNAKE_CASE_						 = True
				if checkpoint_name is None:
						SCREAMING_SNAKE_CASE_						 = list(tokenizer_class.max_model_input_sizes.keys()    )
				else:
						SCREAMING_SNAKE_CASE_						 = [checkpoint_name]

				logger.info(F'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'    )

				for checkpoint in checkpoint_names:
						logger.info(F'Loading {tokenizer_class.__class__.__name__} {checkpoint}'    )

						# Load tokenizer
						SCREAMING_SNAKE_CASE_						 = tokenizer_class.from_pretrained(__a,							force_download=__a    )

						# Save fast tokenizer
						logger.info(F'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'    )

						# For organization names we create sub-directories
						if "/" in checkpoint:
								SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_						 = checkpoint.split('''/'''    )
								SCREAMING_SNAKE_CASE_						 = os.path.join(__a,							__a    )
						elif add_prefix:
								SCREAMING_SNAKE_CASE_						 = checkpoint
								SCREAMING_SNAKE_CASE_						 = dump_path
						else:
								SCREAMING_SNAKE_CASE_						 = None
								SCREAMING_SNAKE_CASE_						 = dump_path

						logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'    )

						if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()    )[0]:
								SCREAMING_SNAKE_CASE_						 = list(tokenizer.pretrained_vocab_files_map.values()    )[0][checkpoint]
								SCREAMING_SNAKE_CASE_						 = file_path.split(__a    )[-1][0]
								if next_char == "/":
										SCREAMING_SNAKE_CASE_						 = os.path.join(__a,							__a    )
										SCREAMING_SNAKE_CASE_						 = None

								logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'    )

						SCREAMING_SNAKE_CASE_						 = tokenizer.save_pretrained(
						    __a,							legacy_format=__a,							filename_prefix=__a    )
						logger.info(F'=> File names {file_names}'    )

						for file_name in file_names:
								if not file_name.endswith('''tokenizer.json'''    ):
										os.remove(__a    )
										logger.info(F'=> removing {file_name}'    )


if __name__ == "__main__":
						lowerCAmelCase__							=				argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '--dump_path', default=None, type=str, required=True, help='Path to output generated fast tokenizer files.'
						)
						parser.add_argument(
						    '--tokenizer_name',
						    default=None,
						    type=str,
						    help=(
						        f'''Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '''
						        'download and convert all the checkpoints from AWS.'
						    ),
						)
						parser.add_argument(
						    '--checkpoint_name',
						    default=None,
						    type=str,
						    help='Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.',
						)
						parser.add_argument(
						    '--force_download',
						    action='store_true',
						    help='Re-download checkpoints.',
						)
						lowerCAmelCase__							=				parser.parse_args()

						convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)

"""simple docstring"""




import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
						from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
						import torch_xla.core.xla_model as xm



def        _lowerCamelCase ( __a    ):
		if is_torch_version('''<''',							'''2.0.0'''    ) or not hasattr(__a,							'''_dynamo'''    ):
				return False
		return isinstance(__a,							torch._dynamo.eval_frame.OptimizedModule    )



def        _lowerCamelCase ( __a,							__a = True    ):
		SCREAMING_SNAKE_CASE_						 = (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

		SCREAMING_SNAKE_CASE_						 = is_compiled_module(__a    )
		if is_compiled:
				SCREAMING_SNAKE_CASE_						 = model
				SCREAMING_SNAKE_CASE_						 = model._orig_mod

		if is_deepspeed_available():
				options += (DeepSpeedEngine,)

		while isinstance(__a,							__a    ):
				SCREAMING_SNAKE_CASE_						 = model.module

		if not keep_fpaa_wrapper:
				SCREAMING_SNAKE_CASE_						 = getattr(__a,							'''forward'''    )
				SCREAMING_SNAKE_CASE_						 = model.__dict__.pop('''_original_forward''',							__a    )
				if original_forward is not None:
						while hasattr(__a,							'''__wrapped__'''    ):
								SCREAMING_SNAKE_CASE_						 = forward.__wrapped__
								if forward == original_forward:
										break
						SCREAMING_SNAKE_CASE_						 = forward
				if getattr(__a,							'''_converted_to_transformer_engine''',							__a    ):
						convert_model(__a,							to_transformer_engine=__a    )

		if is_compiled:
				SCREAMING_SNAKE_CASE_						 = model
				SCREAMING_SNAKE_CASE_						 = compiled_model

		return model



def        _lowerCamelCase ( ):
		PartialState().wait_for_everyone()



def        _lowerCamelCase ( __a,							__a    ):
		if PartialState().distributed_type == DistributedType.TPU:
				xm.save(__a,							__a    )
		elif PartialState().local_process_index == 0:
				torch.save(__a,							__a    )



@contextmanager
def        _lowerCamelCase ( **__a    ):
		for key, value in kwargs.items():
				SCREAMING_SNAKE_CASE_						 = str(__a    )

		yield

		for key in kwargs:
				if key.upper() in os.environ:
						del os.environ[key.upper()]



def        _lowerCamelCase ( __a    ):
		if not hasattr(__a,							'''__qualname__'''    ) and not hasattr(__a,							'''__name__'''    ):
				SCREAMING_SNAKE_CASE_						 = getattr(__a,							'''__class__''',							__a    )
		if hasattr(__a,							'''__qualname__'''    ):
				return obj.__qualname__
		if hasattr(__a,							'''__name__'''    ):
				return obj.__name__
		return str(__a    )



def        _lowerCamelCase ( __a,							__a    ):
		for key, value in source.items():
				if isinstance(__a,							__a    ):
						SCREAMING_SNAKE_CASE_						 = destination.setdefault(__a,							{}    )
						merge_dicts(__a,							__a    )
				else:
						SCREAMING_SNAKE_CASE_						 = value

		return destination



def        _lowerCamelCase ( __a = None    ):
		if port is None:
				SCREAMING_SNAKE_CASE_						 = 29_500
		with socket.socket(socket.AF_INET,							socket.SOCK_STREAM    ) as s:
				return s.connect_ex(('''localhost''', port)    ) == 0