





import logging
import os
import threading
import time


try:
      import warnings
except ImportError:
      lowerCAmelCase_						   =    None

try:
      import msvcrt
except ImportError:
      lowerCAmelCase_						   =    None

try:
      import fcntl
except ImportError:
      lowerCAmelCase_						   =    None


# Backward compatibility
# ------------------------------------------------
try:
      TimeoutError
except NameError:
      lowerCAmelCase_						   =    OSError


# Data
# ------------------------------------------------
lowerCAmelCase_						   =    [
    "Timeout",
    "BaseFileLock",
    "WindowsFileLock",
    "UnixFileLock",
    "SoftFileLock",
    "FileLock",
]

lowerCAmelCase_						   =    "3.0.12"


lowerCAmelCase_						   =    None


def     lowerCamelCase_	(      )->		Tuple:
  global _logger
  _snake_case   :				str														=	_logger or logging.getLogger(__name__				)
  return _logger



class        lowerCAmelCase_    (			_UpperCAmelCase							):


  '''simple docstring'''







  def __init__(      self      :					List[str]      ,				UpperCamelCase      :					int							):


    '''simple docstring'''





    _snake_case   :				int														=	lock_file
    return None







  def __str__(      self      :					str							):


    '''simple docstring'''





    _snake_case   :				Any														=	f"""The file lock '{self.lock_file}' could not be acquired."""
    return temp




class        lowerCAmelCase_    :


  '''simple docstring'''







  def __init__(      self      :					Optional[int]      ,				UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				List[str]														=	lock
    return None







  def __enter__(      self      :					Optional[Any]							):


    '''simple docstring'''





    return self.lock







  def __exit__(      self      :					Optional[Any]      ,				UpperCamelCase      :					Dict      ,				UpperCamelCase      :					Dict      ,				UpperCamelCase      :					Tuple							):


    '''simple docstring'''





    self.lock.release()
    return None




class        lowerCAmelCase_    :


  '''simple docstring'''







  def __init__(      self      :					List[Any]      ,				UpperCamelCase      :					Any      ,				UpperCamelCase      :					Optional[int]=-1      ,				UpperCamelCase      :					str=None							):


    '''simple docstring'''





    _snake_case   :				Dict														=	max_filename_length if max_filename_length is not None else 2_55
    # Hash the filename if it's too long
    _snake_case   :				List[str]														=	self.hash_filename_if_too_long(A_      ,				A_							)
    # The path to the lock file.
    _snake_case   :				Any														=	lock_file

    # The file descriptor for the *_lock_file* as it is returned by the
    # os.open() function.
    # This file lock is only NOT None, if the object currently holds the
    # lock.
    _snake_case   :				List[Any]														=	None

    # The default timeout value.
    _snake_case   :				Optional[Any]														=	timeout

    # We use this lock primarily for the lock counter.
    _snake_case   :				int														=	threading.Lock()

    # The lock counter is used for implementing the nested locking
    # mechanism. Whenever the lock is acquired, the counter is increased and
    # the lock is only released, when this value is 0 again.
    _snake_case   :				Tuple														=	0
    return None







  @property
  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    return self._lock_file







  @property
  def UpperCamelCase_ (      self      :					Optional[Any]							):


    '''simple docstring'''





    return self._timeout







  @timeout.setter
  def UpperCamelCase_ (      self      :					List[str]      ,				UpperCamelCase      :					int							):


    '''simple docstring'''





    _snake_case   :				int														=	float(A_							)
    return None







  def UpperCamelCase_ (      self      :					Optional[Any]							):


    '''simple docstring'''





    raise NotImplementedError()







  def UpperCamelCase_ (      self      :					Union[str, Any]							):


    '''simple docstring'''





    raise NotImplementedError()







  @property
  def UpperCamelCase_ (      self      :					str							):


    '''simple docstring'''





    return self._lock_file_fd is not None







  def UpperCamelCase_ (      self      :					Tuple      ,				UpperCamelCase      :					Union[str, Any]=None      ,				UpperCamelCase      :					List[Any]=0.05							):


    '''simple docstring'''





    if timeout is None:
      _snake_case   :				Union[str, Any]														=	self.timeout

    # Increment the number right at the beginning.
    # We can still undo it, if something fails.
    with self._thread_lock:
      self._lock_counter += 1

    _snake_case   :				Optional[int]														=	id(self							)
    _snake_case   :				int														=	self._lock_file
    _snake_case   :				str														=	time.time()
    try:
      while True:
        with self._thread_lock:
          if not self.is_locked:
            logger().debug(f"""Attempting to acquire lock {lock_id} on {lock_filename}"""							)
            self._acquire()

        if self.is_locked:
          logger().debug(f"""Lock {lock_id} acquired on {lock_filename}"""							)
          break
        elif timeout >= 0 and time.time() - start_time > timeout:
          logger().debug(f"""Timeout on acquiring lock {lock_id} on {lock_filename}"""							)
          raise Timeout(self._lock_file							)
        else:
          logger().debug(
              f"""Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ..."""							)
          time.sleep(A_							)
    except:  # noqa
      # Something did go wrong, so decrement the counter.
      with self._thread_lock:
        _snake_case   :				int														=	max(0      ,				self._lock_counter - 1							)

      raise
    return _Acquire_ReturnProxy(lock=self							)







  def UpperCamelCase_ (      self      :					str      ,				UpperCamelCase      :					List[str]=False							):


    '''simple docstring'''





    with self._thread_lock:
      if self.is_locked:
        self._lock_counter -= 1

        if self._lock_counter == 0 or force:
          _snake_case   :				Tuple														=	id(self							)
          _snake_case   :				int														=	self._lock_file

          logger().debug(f"""Attempting to release lock {lock_id} on {lock_filename}"""							)
          self._release()
          _snake_case   :				Union[str, Any]														=	0
          logger().debug(f"""Lock {lock_id} released on {lock_filename}"""							)

    return None







  def __enter__(      self      :					int							):


    '''simple docstring'''





    self.acquire()
    return self







  def __exit__(      self      :					Optional[Any]      ,				UpperCamelCase      :					Optional[int]      ,				UpperCamelCase      :					List[Any]      ,				UpperCamelCase      :					List[str]							):


    '''simple docstring'''





    self.release()
    return None







  def __del__(      self      :					str							):


    '''simple docstring'''





    self.release(force=A_							)
    return None







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				UpperCamelCase      :					str      ,				UpperCamelCase      :					int							):


    '''simple docstring'''





    _snake_case   :				Any														=	os.path.basename(A_							)
    if len(A_							) > max_length and max_length > 0:
      _snake_case   :				List[str]														=	os.path.dirname(A_							)
      _snake_case   :				Optional[int]														=	str(hash(A_							)							)
      _snake_case   :				int														=	filename[: max_length - len(A_							) - 8] + '...' + hashed_filename + '.lock'
      return os.path.join(A_      ,				A_							)
    else:
      return path




class        lowerCAmelCase_    (			_UpperCAmelCase							):


  '''simple docstring'''







  def __init__(      self      :					Optional[int]      ,				UpperCamelCase      :					Optional[Any]      ,				UpperCamelCase      :					Optional[int]=-1      ,				UpperCamelCase      :					str=None							):


    '''simple docstring'''





    from .file_utils import relative_to_absolute_path

    super().__init__(A_      ,				timeout=A_      ,				max_filename_length=A_							)
    _snake_case   :				Any														=	'\\\\?\\' + relative_to_absolute_path(self.lock_file							)







  def UpperCamelCase_ (      self      :					Dict							):


    '''simple docstring'''





    _snake_case   :				List[Any]														=	os.O_RDWR | os.O_CREAT | os.O_TRUNC

    try:
      _snake_case   :				Union[str, Any]														=	os.open(self._lock_file      ,				A_							)
    except OSError:
      pass
    else:
      try:
        msvcrt.locking(A_      ,				msvcrt.LK_NBLCK      ,				1							)
      except OSError:
        os.close(A_							)
      else:
        _snake_case   :				List[str]														=	fd
    return None







  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    _snake_case   :				Optional[int]														=	self._lock_file_fd
    _snake_case   :				Optional[Any]														=	None
    msvcrt.locking(A_      ,				msvcrt.LK_UNLCK      ,				1							)
    os.close(A_							)

    try:
      os.remove(self._lock_file							)
    # Probably another instance of the application
    # that acquired the file lock.
    except OSError:
      pass
    return None




class        lowerCAmelCase_    (			_UpperCAmelCase							):


  '''simple docstring'''







  def __init__(      self      :					str      ,				UpperCamelCase      :					Any      ,				UpperCamelCase      :					Optional[Any]=-1      ,				UpperCamelCase      :					Tuple=None							):


    '''simple docstring'''





    _snake_case   :				List[str]														=	os.statvfs(os.path.dirname(A_							)							).f_namemax
    super().__init__(A_      ,				timeout=A_      ,				max_filename_length=A_							)







  def UpperCamelCase_ (      self      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				Dict														=	os.O_RDWR | os.O_CREAT | os.O_TRUNC
    _snake_case   :				str														=	os.open(self._lock_file      ,				A_							)

    try:
      fcntl.flock(A_      ,				fcntl.LOCK_EX | fcntl.LOCK_NB							)
    except OSError:
      os.close(A_							)
    else:
      _snake_case   :				Union[str, Any]														=	fd
    return None







  def UpperCamelCase_ (      self      :					str							):


    '''simple docstring'''





    _snake_case   :				List[Any]														=	self._lock_file_fd
    _snake_case   :				Dict														=	None
    fcntl.flock(A_      ,				fcntl.LOCK_UN							)
    os.close(A_							)
    return None




class        lowerCAmelCase_    (			_UpperCAmelCase							):


  '''simple docstring'''







  def UpperCamelCase_ (      self      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				List[Any]														=	os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
    try:
      _snake_case   :				str														=	os.open(self._lock_file      ,				A_							)
    except OSError:
      pass
    else:
      _snake_case   :				List[Any]														=	fd
    return None







  def UpperCamelCase_ (      self      :					Dict							):


    '''simple docstring'''





    os.close(self._lock_file_fd							)
    _snake_case   :				List[Any]														=	None

    try:
      os.remove(self._lock_file							)
    # The file is already deleted and that's what we want.
    except OSError:
      pass
    return None




lowerCAmelCase_						   =    None

if msvcrt:
      lowerCAmelCase_						   =    WindowsFileLock
elif fcntl:
      lowerCAmelCase_						   =    UnixFileLock
else:
      lowerCAmelCase_						   =    SoftFileLock

      if warnings is not None:
            warnings.warn("""only soft file lock is available""")











import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

lowerCAmelCase_						   =    logging.get_logger(__name__)


lowerCAmelCase_						   =    {name: getattr(transformers, name + """Fast""") for name in SLOW_TO_FAST_CONVERTERS}


def     lowerCamelCase_	(      lowerCAmelCase:	List[Any]    ,							lowerCAmelCase:	Optional[Any]    ,							lowerCAmelCase:	Dict    ,							lowerCAmelCase:	Union[str, Any]				)->		Optional[int]:
  if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
    raise ValueError(F"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()				)}."""				)

  if tokenizer_name is None:
    _snake_case   :				Tuple														=	TOKENIZER_CLASSES
  else:
    _snake_case   :				Union[str, Any]														=	{tokenizer_name: getattr(lowerCAmelCase    ,							tokenizer_name + 'Fast'				)}

  logger.info(F"""Loading tokenizer classes: {tokenizer_names}"""				)

  for tokenizer_name in tokenizer_names:
    _snake_case   :				Dict														=	TOKENIZER_CLASSES[tokenizer_name]

    _snake_case   :				Optional[Any]														=	True
    if checkpoint_name is None:
      _snake_case   :				Union[str, Any]														=	list(tokenizer_class.max_model_input_sizes.keys()				)
    else:
      _snake_case   :				Optional[int]														=	[checkpoint_name]

    logger.info(F"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""				)

    for checkpoint in checkpoint_names:
      logger.info(F"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""				)

      # Load tokenizer
      _snake_case   :				str														=	tokenizer_class.from_pretrained(lowerCAmelCase    ,							force_download=lowerCAmelCase				)

      # Save fast tokenizer
      logger.info(F"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""				)

      # For organization names we create sub-directories
      if "/" in checkpoint:
        _snake_case							,					_snake_case   :				Tuple														=	checkpoint.split('/'				)
        _snake_case   :				int														=	os.path.join(lowerCAmelCase    ,							lowerCAmelCase				)
      elif add_prefix:
        _snake_case   :				Dict														=	checkpoint
        _snake_case   :				Optional[Any]														=	dump_path
      else:
        _snake_case   :				str														=	None
        _snake_case   :				Union[str, Any]														=	dump_path

      logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""				)

      if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()				)[0]:
        _snake_case   :				Optional[Any]														=	list(tokenizer.pretrained_vocab_files_map.values()				)[0][checkpoint]
        _snake_case   :				Optional[int]														=	file_path.split(lowerCAmelCase				)[-1][0]
        if next_char == "/":
          _snake_case   :				Union[str, Any]														=	os.path.join(lowerCAmelCase    ,							lowerCAmelCase				)
          _snake_case   :				str														=	None

        logger.info(F"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""				)

      _snake_case   :				Optional[int]														=	tokenizer.save_pretrained(
          lowerCAmelCase    ,							legacy_format=lowerCAmelCase    ,							filename_prefix=lowerCAmelCase				)
      logger.info(F"""=> File names {file_names}"""				)

      for file_name in file_names:
        if not file_name.endswith('tokenizer.json'				):
          os.remove(lowerCAmelCase				)
          logger.info(F"""=> removing {file_name}"""				)


if __name__ == "__main__":
      lowerCAmelCase_						   =    argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--dump_path""", default=None, type=str, required=True, help="""Path to output generated fast tokenizer files."""
      )
      parser.add_argument(
          """--tokenizer_name""",
          default=None,
          type=str,
          help=(
              F"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
              """download and convert all the checkpoints from AWS."""
          ),
      )
      parser.add_argument(
          """--checkpoint_name""",
          default=None,
          type=str,
          help="""Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.""",
      )
      parser.add_argument(
          """--force_download""",
          action="""store_true""",
          help="""Re-download checkpoints.""",
      )
      lowerCAmelCase_						   =    parser.parse_args()

      convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)




