import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase					:   Union[str, Any]							=		logging.get_logger(__name__)

_lowerCamelCase					:   Optional[int]							=		'''▁'''

_lowerCamelCase					:   Any							=		{'''vocab_file''': '''sentencepiece.bpe.model'''}

_lowerCamelCase					:   List[str]							=		{
    '''vocab_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model''',
    }
}

_lowerCamelCase					:   Dict							=		{
    '''facebook/xglm-564M''': 20_48,
}


class 						lowercase    (     a     ):
							lowercase__						:       Dict								=							VOCAB_FILES_NAMES
							lowercase__						:       int								=							PRETRAINED_VOCAB_FILES_MAP
							lowercase__						:       Optional[Any]								=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__						:       Union[str, Any]								=							["""input_ids""", """attention_mask"""]




							def __init__(						self   :					Optional[int]	,							_UpperCamelCase   :					Optional[Any]	,							_UpperCamelCase   :					Dict="<s>"	,							_UpperCamelCase   :					str="</s>"	,							_UpperCamelCase   :					Union[str, Any]="</s>"	,							_UpperCamelCase   :					Tuple="<s>"	,							_UpperCamelCase   :					List[str]="<unk>"	,							_UpperCamelCase   :					Optional[Any]="<pad>"	,							_UpperCamelCase   :					Optional[Dict[str, Any]] = None	,							**_UpperCamelCase   :					List[str]	,							)					->				None:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       {} if sp_model_kwargs is None else sp_model_kwargs

												# Compatibility with the original tokenizer
												SCREAMING_SNAKE_CASE    			=       7
												SCREAMING_SNAKE_CASE    			=       [F"<madeupword{i}>" for i in range(self.num_madeup_words      )]

												SCREAMING_SNAKE_CASE    			=       kwargs.get("additional_special_tokens"	,							[]      )
												kwargs["additional_special_tokens"] += [
												    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
												]

												super().__init__(
												    bos_token=_UpperCamelCase	,							eos_token=_UpperCamelCase	,							unk_token=_UpperCamelCase	,							sep_token=_UpperCamelCase	,							cls_token=_UpperCamelCase	,							pad_token=_UpperCamelCase	,							sp_model_kwargs=self.sp_model_kwargs	,							**_UpperCamelCase	,							)

												SCREAMING_SNAKE_CASE    			=       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
												self.sp_model.Load(str(_UpperCamelCase      )      )
												SCREAMING_SNAKE_CASE    			=       vocab_file

												# Original fairseq vocab and spm vocab must be "aligned":
												# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
												# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
												# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
												# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

												# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
												SCREAMING_SNAKE_CASE    			=       1

												# Mimic fairseq token-to-id alignment for the first 4 token
												SCREAMING_SNAKE_CASE    			=       {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

												SCREAMING_SNAKE_CASE    			=       len(self.sp_model      )
												SCREAMING_SNAKE_CASE    			=       {F"<madeupword{i}>": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words      )}
												self.fairseq_tokens_to_ids.update(_UpperCamelCase      )

												SCREAMING_SNAKE_CASE    			=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}




							def __getstate__(						self   :					int      )					->				Any:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       self.__dict__.copy()
												SCREAMING_SNAKE_CASE    			=       None
												SCREAMING_SNAKE_CASE    			=       self.sp_model.serialized_model_proto()
												return state




							def __setstate__(						self   :					Union[str, Any]	,							_UpperCamelCase   :					Optional[Any]      )					->				Any:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       d

												# for backward compatibility
												if not hasattr(self	,							"sp_model_kwargs"      ):
																	SCREAMING_SNAKE_CASE    			=       {}

												SCREAMING_SNAKE_CASE    			=       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
												self.sp_model.LoadFromSerializedProto(self.sp_model_proto      )




							def 	__snake_case(						self   :					List[str]	,							_UpperCamelCase   :					List[int]	,							_UpperCamelCase   :					Optional[List[int]] = None      )					->				List[int]:



												'''simple docstring'''

												if token_ids_a is None:
																	return [self.sep_token_id] + token_ids_a
												SCREAMING_SNAKE_CASE    			=       [self.sep_token_id]
												return sep + token_ids_a + sep + sep + token_ids_a




							def 	__snake_case(						self   :					Optional[Any]	,							_UpperCamelCase   :					List[int]	,							_UpperCamelCase   :					Optional[List[int]] = None	,							_UpperCamelCase   :					bool = False      )					->				List[int]:



												'''simple docstring'''

												if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=_UpperCamelCase	,							token_ids_a=_UpperCamelCase	,							already_has_special_tokens=_UpperCamelCase      )

												if token_ids_a is None:
																	return [1] + ([0] * len(_UpperCamelCase      ))
												return [1] + ([0] * len(_UpperCamelCase      )) + [1, 1] + ([0] * len(_UpperCamelCase      ))




							def 	__snake_case(						self   :					List[str]	,							_UpperCamelCase   :					List[int]	,							_UpperCamelCase   :					Optional[List[int]] = None      )					->				List[int]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       [self.sep_token_id]

												if token_ids_a is None:
																	return len(sep + token_ids_a      ) * [0]
												return len(sep + token_ids_a + sep + sep + token_ids_a      ) * [0]




							@property
							def 	__snake_case(						self   :					List[Any]      )					->				Union[str, Any]:



												'''simple docstring'''

												return len(self.sp_model      ) + self.fairseq_offset + self.num_madeup_words




							def 	__snake_case(						self   :					List[Any]      )					->				Optional[Any]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       {self.convert_ids_to_tokens(_UpperCamelCase      ): i for i in range(self.vocab_size      )}
												vocab.update(self.added_tokens_encoder      )
												return vocab




							def 	__snake_case(						self   :					List[str]	,							_UpperCamelCase   :					str      )					->				List[str]:



												'''simple docstring'''

												return self.sp_model.encode(_UpperCamelCase	,							out_type=_UpperCamelCase      )




							def 	__snake_case(						self   :					str	,							_UpperCamelCase   :					Tuple      )					->				Optional[int]:



												'''simple docstring'''

												if token in self.fairseq_tokens_to_ids:
																	return self.fairseq_tokens_to_ids[token]
												SCREAMING_SNAKE_CASE    			=       self.sp_model.PieceToId(_UpperCamelCase      )

												# Need to return unknown token if the SP model returned 0
												return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




							def 	__snake_case(						self   :					Any	,							_UpperCamelCase   :					Dict      )					->				Dict:



												'''simple docstring'''

												if index in self.fairseq_ids_to_tokens:
																	return self.fairseq_ids_to_tokens[index]
												return self.sp_model.IdToPiece(index - self.fairseq_offset      )




							def 	__snake_case(						self   :					List[Any]	,							_UpperCamelCase   :					Dict      )					->				List[str]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       "".join(_UpperCamelCase      ).replace(_UpperCamelCase	,							" "      ).strip()
												return out_string





							def 	__snake_case(						self   :					Dict	,							_UpperCamelCase   :					str	,							_UpperCamelCase   :					Optional[str] = None      )					->				Tuple[str]:



												'''simple docstring'''

												if not os.path.isdir(_UpperCamelCase      ):
																	logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
																	return
												SCREAMING_SNAKE_CASE    			=       os.path.join(
												    _UpperCamelCase	,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )

												if os.path.abspath(self.vocab_file      ) != os.path.abspath(_UpperCamelCase      ) and os.path.isfile(self.vocab_file      ):
																	copyfile(self.vocab_file	,							_UpperCamelCase      )
												elif not os.path.isfile(self.vocab_file      ):
																	with open(_UpperCamelCase	,							"wb"      ) as fi:
																						SCREAMING_SNAKE_CASE    			=       self.sp_model.serialized_model_proto()
																						fi.write(_UpperCamelCase      )

												return (out_vocab_file,)





import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


_lowerCamelCase					:   str							=		threading.Lock()
_lowerCamelCase					:   Optional[logging.Handler]							=		None

_lowerCamelCase					:   Any							=		{
    '''debug''': logging.DEBUG,
    '''info''': logging.INFO,
    '''warning''': logging.WARNING,
    '''error''': logging.ERROR,
    '''critical''': logging.CRITICAL,
}

_lowerCamelCase					:   Union[str, Any]							=		logging.WARNING

_lowerCamelCase					:   List[Any]							=		True





def 	__lowerCamelCase							():
					SCREAMING_SNAKE_CASE    			=       os.getenv("TRANSFORMERS_VERBOSITY"      , UpperCAmelCase__  )
					if env_level_str:
										if env_level_str in log_levels:
															return log_levels[env_level_str]
										else:
															logging.getLogger().warning(
															    F"Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, "
															    F"has to be one of: { ', '.join(log_levels.keys()  ) }"  )
					return _default_log_level





def 	__lowerCamelCase							():
					return __name__.split("."  )[0]





def 	__lowerCamelCase							():
					return logging.getLogger(_get_library_name()  )





def 	__lowerCamelCase							():
					global _default_handler

					with _lock:
										if _default_handler:
															# This library has already configured the library root logger.
															return
										SCREAMING_SNAKE_CASE    			=       logging.StreamHandler()  # Set sys.stderr as stream.
										SCREAMING_SNAKE_CASE    			=       sys.stderr.flush

										# Apply our default configuration to the library root logger.
										SCREAMING_SNAKE_CASE    			=       _get_library_root_logger()
										library_root_logger.addHandler(_default_handler  )
										library_root_logger.setLevel(_get_default_logging_level()  )
										SCREAMING_SNAKE_CASE    			=       False





def 	__lowerCamelCase							():
					global _default_handler

					with _lock:
										if not _default_handler:
															return

										SCREAMING_SNAKE_CASE    			=       _get_library_root_logger()
										library_root_logger.removeHandler(_default_handler  )
										library_root_logger.setLevel(logging.NOTSET  )
										SCREAMING_SNAKE_CASE    			=       None





def 	__lowerCamelCase							():
					return log_levels





def 	__lowerCamelCase							(UpperCAmelCase__     :							Optional[str] = None  ):

					if name is None:
										SCREAMING_SNAKE_CASE    			=       _get_library_name()

					_configure_library_root_logger()
					return logging.getLogger(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					_configure_library_root_logger()
					return _get_library_root_logger().getEffectiveLevel()





def 	__lowerCamelCase							(UpperCAmelCase__     :							int  ):
					_configure_library_root_logger()
					_get_library_root_logger().setLevel(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					return set_verbosity(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					return set_verbosity(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					return set_verbosity(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					return set_verbosity(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					_configure_library_root_logger()

					assert _default_handler is not None
					_get_library_root_logger().removeHandler(_default_handler  )





def 	__lowerCamelCase							():
					_configure_library_root_logger()

					assert _default_handler is not None
					_get_library_root_logger().addHandler(_default_handler  )





def 	__lowerCamelCase							(UpperCAmelCase__     :							logging.Handler  ):
					_configure_library_root_logger()

					assert handler is not None
					_get_library_root_logger().addHandler(UpperCAmelCase__  )





def 	__lowerCamelCase							(UpperCAmelCase__     :							logging.Handler  ):
					_configure_library_root_logger()

					assert handler is not None and handler not in _get_library_root_logger().handlers
					_get_library_root_logger().removeHandler(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					_configure_library_root_logger()
					SCREAMING_SNAKE_CASE    			=       False





def 	__lowerCamelCase							():
					_configure_library_root_logger()
					SCREAMING_SNAKE_CASE    			=       True





def 	__lowerCamelCase							():
					SCREAMING_SNAKE_CASE    			=       _get_library_root_logger().handlers

					for handler in handlers:
										SCREAMING_SNAKE_CASE    			=       logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"  )
										handler.setFormatter(UpperCAmelCase__  )





def 	__lowerCamelCase							():
					SCREAMING_SNAKE_CASE    			=       _get_library_root_logger().handlers

					for handler in handlers:
										handler.setFormatter(UpperCAmelCase__  )





def 	__lowerCamelCase							(self     :							str      , *UpperCAmelCase__     :							Union[str, Any]      , **UpperCAmelCase__     :							List[str]  ):
					SCREAMING_SNAKE_CASE    			=       os.getenv("TRANSFORMERS_NO_ADVISORY_WARNINGS"      , UpperCAmelCase__  )
					if no_advisory_warnings:
										return
					self.warning(*UpperCAmelCase__      , **UpperCAmelCase__  )


_lowerCamelCase					:   str							=		warning_advice





@functools.lru_cache(UpperCAmelCase__  )
def 	__lowerCamelCase							(self     :							List[str]      , *UpperCAmelCase__     :							Tuple      , **UpperCAmelCase__     :							int  ):
					self.warning(*UpperCAmelCase__      , **UpperCAmelCase__  )


_lowerCamelCase					:   Dict							=		warning_once


class 						lowercase    :




							def __init__(						self   :					List[Any]	,							*_UpperCamelCase   :					Union[str, Any]	,							**_UpperCamelCase   :					str      )					->				List[Any]:  # pylint: disable=unused-argument



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       args[0] if args else None




							def __iter__(						self   :					Optional[Any]      )					->				str:



												'''simple docstring'''

												return iter(self._iterator      )




							def __getattr__(						self   :					List[str]	,							_UpperCamelCase   :					Any      )					->				List[Any]:



												'''simple docstring'''

												def empty_fn(*_UpperCamelCase   :					List[str]	,							**_UpperCamelCase   :					Union[str, Any]      ):  # pylint: disable=unused-argument
																	return

												return empty_fn




							def __enter__(						self   :					Any      )					->				Optional[Any]:



												'''simple docstring'''

												return self





							def __exit__(						self   :					Optional[Any]	,							_UpperCamelCase   :					Tuple	,							_UpperCamelCase   :					str	,							_UpperCamelCase   :					List[str]      )					->				Union[str, Any]:



												'''simple docstring'''

												return



class 						lowercase    :




							def __call__(						self   :					Union[str, Any]	,							*_UpperCamelCase   :					Optional[Any]	,							**_UpperCamelCase   :					Tuple      )					->				Tuple:



												'''simple docstring'''

												if _tqdm_active:
																	return tqdm_lib.tqdm(*_UpperCamelCase	,							**_UpperCamelCase      )
												else:
																	return EmptyTqdm(*_UpperCamelCase	,							**_UpperCamelCase      )




							def 	__snake_case(						self   :					Dict	,							*_UpperCamelCase   :					Dict	,							**_UpperCamelCase   :					Union[str, Any]      )					->				Union[str, Any]:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE    			=       None
												if _tqdm_active:
																	return tqdm_lib.tqdm.set_lock(*_UpperCamelCase	,							**_UpperCamelCase      )





							def 	__snake_case(						self   :					Optional[int]      )					->				Union[str, Any]:



												'''simple docstring'''

												if _tqdm_active:
																	return tqdm_lib.tqdm.get_lock()



_lowerCamelCase					:   Union[str, Any]							=		_tqdm_cls()





def 	__lowerCamelCase							():
					global _tqdm_active
					return bool(_tqdm_active  )





def 	__lowerCamelCase							():
					global _tqdm_active
					SCREAMING_SNAKE_CASE    			=       True
					hf_hub_utils.enable_progress_bars()





def 	__lowerCamelCase							():
					global _tqdm_active
					SCREAMING_SNAKE_CASE    			=       False
					hf_hub_utils.disable_progress_bars()




