






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase : Any						   =       logging.get_logger(__name__)
lowerCamelCase : Tuple						   =       {'''vocab_file''': '''spiece.model'''}

lowerCamelCase : str						   =       {
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

lowerCamelCase : Any						   =       {
    '''albert-base-v1''': 5_12,
    '''albert-large-v1''': 5_12,
    '''albert-xlarge-v1''': 5_12,
    '''albert-xxlarge-v1''': 5_12,
    '''albert-base-v2''': 5_12,
    '''albert-large-v2''': 5_12,
    '''albert-xlarge-v2''': 5_12,
    '''albert-xxlarge-v2''': 5_12,
}

lowerCamelCase : Optional[Any]						   =       '''▁'''





class 		lowerCAmelCase     ( __a			):

					'''simple docstring'''

					_A						:			List[Any]									=      VOCAB_FILES_NAMES
					_A						:			int									=      PRETRAINED_VOCAB_FILES_MAP
					_A						:			Dict									=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



					def __init__(  self				:	Union[str, Any]		,     __a				:	str		,     __a				:	List[Any]=True		,     __a				:	List[Any]=True		,     __a				:	List[str]=False		,     __a				:	Any="[CLS]"		,     __a				:	Tuple="[SEP]"		,     __a				:	Tuple="<unk>"		,     __a				:	str="[SEP]"		,     __a				:	Tuple="<pad>"		,     __a				:	Tuple="[CLS]"		,     __a				:	Dict="[MASK]"		,     __a				:	Optional[Dict[str, Any]] = None		,     **__a				:	List[str]		,     )			->  None:


												"""simple docstring"""

												__lowercase      :							Dict				=     (
												    AddedToken(__a		,     lstrip=__a		,     rstrip=__a		,     normalized=__a							)
												    if isinstance(__a		,     __a							)
												    else mask_token
												)

												__lowercase      :							int				=     {} if sp_model_kwargs is None else sp_model_kwargs

												super().__init__(
												    do_lower_case=__a		,     remove_space=__a		,     keep_accents=__a		,     bos_token=__a		,     eos_token=__a		,     unk_token=__a		,     sep_token=__a		,     pad_token=__a		,     cls_token=__a		,     mask_token=__a		,     sp_model_kwargs=self.sp_model_kwargs		,     **__a		,     )

												__lowercase      :							List[Any]				=     do_lower_case
												__lowercase      :							str				=     remove_space
												__lowercase      :							Optional[Any]				=     keep_accents
												__lowercase      :							Any				=     vocab_file

												__lowercase      :							Any				=     spm.SentencePieceProcessor(**self.sp_model_kwargs							)
												self.sp_model.Load(__a							)



					@property
					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Any:


												"""simple docstring"""

												return len(self.sp_model							)



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							Optional[Any]				=     {self.convert_ids_to_tokens(__a							): i for i in range(self.vocab_size							)}
												vocab.update(self.added_tokens_encoder							)
												return vocab



					def __getstate__(  self				:	Any							)			->  Dict:


												"""simple docstring"""

												__lowercase      :							Any				=     self.__dict__.copy()
												__lowercase      :							Dict				=     None
												return state



					def __setstate__(  self				:	Dict		,     __a				:	str							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							int				=     d

												# for backward compatibility
												if not hasattr(self		,     """sp_model_kwargs"""							):
																			__lowercase      :							List[str]				=     {}

												__lowercase      :							Tuple				=     spm.SentencePieceProcessor(**self.sp_model_kwargs							)
												self.sp_model.Load(self.vocab_file							)



					def       lowerCAmelCase				(  self				:	int		,     __a				:	Optional[int]							)			->  int:


												"""simple docstring"""

												if self.remove_space:
																			__lowercase      :							Union[str, Any]				=     """ """.join(inputs.strip().split()							)
												else:
																			__lowercase      :							Any				=     inputs
												__lowercase      :							Optional[int]				=     outputs.replace("""``"""		,     """\""""							).replace("""''"""		,     """\""""							)

												if not self.keep_accents:
																			__lowercase      :							Optional[int]				=     unicodedata.normalize("""NFKD"""		,     __a							)
																			__lowercase      :							Optional[Any]				=     """""".join([c for c in outputs if not unicodedata.combining(__a							)]							)
												if self.do_lower_case:
																			__lowercase      :							str				=     outputs.lower()

												return outputs



					def       lowerCAmelCase				(  self				:	Tuple		,     __a				:	str							)			->  List[str]:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     self.preprocess_text(__a							)
												__lowercase      :							Optional[Any]				=     self.sp_model.encode(__a		,     out_type=__a							)
												__lowercase      :							Optional[Any]				=     []
												for piece in pieces:
																			if len(__a							) > 1 and piece[-1] == str(""","""							) and piece[-2].isdigit():
																										__lowercase      :							int				=     self.sp_model.EncodeAsPieces(piece[:-1].replace(__a		,     """"""							)							)
																										if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																	if len(cur_pieces[0]							) == 1:
																																								__lowercase      :							Optional[int]				=     cur_pieces[1:]
																																	else:
																																								__lowercase      :							str				=     cur_pieces[0][1:]
																										cur_pieces.append(piece[-1]							)
																										new_pieces.extend(__a							)
																			else:
																										new_pieces.append(__a							)

												return new_pieces



					def       lowerCAmelCase				(  self				:	Tuple		,     __a				:	Dict							)			->  Optional[Any]:


												"""simple docstring"""

												return self.sp_model.PieceToId(__a							)



					def       lowerCAmelCase				(  self				:	Union[str, Any]		,     __a				:	Union[str, Any]							)			->  List[str]:


												"""simple docstring"""

												return self.sp_model.IdToPiece(__a							)



					def       lowerCAmelCase				(  self				:	Union[str, Any]		,     __a				:	List[Any]							)			->  List[str]:


												"""simple docstring"""

												__lowercase      :							Any				=     []
												__lowercase      :							Dict				=     """"""
												__lowercase      :							str				=     False
												for token in tokens:
																			# make sure that special tokens are not decoded using sentencepiece model
																			if token in self.all_special_tokens:
																										if not prev_is_special:
																																	out_string += " "
																										out_string += self.sp_model.decode(__a							) + token
																										__lowercase      :							Optional[Any]				=     True
																										__lowercase      :							List[str]				=     []
																			else:
																										current_sub_tokens.append(__a							)
																										__lowercase      :							Optional[Any]				=     False
												out_string += self.sp_model.decode(__a							)
												return out_string.strip()



					def       lowerCAmelCase				(  self				:	str		,     __a				:	List[int]		,     __a				:	Optional[List[int]] = None							)			->  List[int]:


												"""simple docstring"""

												__lowercase      :							Optional[int]				=     [self.sep_token_id]
												__lowercase      :							Dict				=     [self.cls_token_id]
												if token_ids_a is None:
																			return cls + token_ids_a + sep
												return cls + token_ids_a + sep + token_ids_a + sep



					def       lowerCAmelCase				(  self				:	int		,     __a				:	List[int]		,     __a				:	Optional[List[int]] = None		,     __a				:	bool = False							)			->  List[int]:


												"""simple docstring"""

												if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=__a		,     token_ids_a=__a		,     already_has_special_tokens=__a							)

												if token_ids_a is not None:
																			return [1] + ([0] * len(__a							)) + [1] + ([0] * len(__a							)) + [1]
												return [1] + ([0] * len(__a							)) + [1]



					def       lowerCAmelCase				(  self				:	Optional[int]		,     __a				:	List[int]		,     __a				:	Optional[List[int]] = None							)			->  List[int]:


												"""simple docstring"""

												__lowercase      :							List[str]				=     [self.sep_token_id]
												__lowercase      :							List[Any]				=     [self.cls_token_id]

												if token_ids_a is None:
																			return len(cls + token_ids_a + sep							) * [0]
												return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]



					def       lowerCAmelCase				(  self				:	Any		,     __a				:	str		,     __a				:	Optional[str] = None							)			->  Tuple[str]:


												"""simple docstring"""

												if not os.path.isdir(__a							):
																			logger.error(F"Vocabulary path ({save_directory}) should be a directory"							)
																			return
												__lowercase      :							Optional[Any]				=     os.path.join(
												    __a		,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

												if os.path.abspath(self.vocab_file							) != os.path.abspath(__a							) and os.path.isfile(self.vocab_file							):
																			copyfile(self.vocab_file		,     __a							)
												elif not os.path.isfile(self.vocab_file							):
																			with open(__a		,     """wb"""							) as fi:
																										__lowercase      :							Tuple				=     self.sp_model.serialized_model_proto()
																										fi.write(__a							)

												return (out_vocab_file,)







# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


lowerCamelCase : int						   =       '''Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine'''
def       snake_case_		(     ):
							__lowercase      :							List[Any]				=     _ask_options(
							    """In which compute environment are you running?"""       ,       ["""This machine""", """AWS (Amazon SageMaker)"""]       ,       _convert_compute_environment       ,       )
							if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
														__lowercase      :							Union[str, Any]				=     get_sagemaker_input()
							else:
														__lowercase      :							str				=     get_cluster_input()
							return config
def       snake_case_		(     lowerCAmelCase_       :						List[str]=None      ):
							if subparsers is not None:
														__lowercase      :							Optional[int]				=     subparsers.add_parser("""config"""       ,       description=lowerCAmelCase_      )
							else:
														__lowercase      :							List[str]				=     argparse.ArgumentParser("""Accelerate config command"""       ,       description=lowerCAmelCase_      )

							parser.add_argument(
							    """--config_file"""       ,       default=lowerCAmelCase_       ,       help=(
							        """The path to use to store the config file. Will default to a file named default_config.yaml in the cache """
							        """location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have """
							        """such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed """
							        """with 'huggingface'."""
							    )       ,       )

							if subparsers is not None:
														parser.set_defaults(func=lowerCAmelCase_      )
							return parser
def       snake_case_		(     lowerCAmelCase_       :						Tuple      ):
							__lowercase      :							Union[str, Any]				=     get_user_input()
							if args.config_file is not None:
														__lowercase      :							List[Any]				=     args.config_file
							else:
														if not os.path.isdir(lowerCAmelCase_      ):
																					os.makedirs(lowerCAmelCase_      )
														__lowercase      :							Any				=     default_yaml_config_file

							if config_file.endswith(""".json"""      ):
														config.to_json_file(lowerCAmelCase_      )
							else:
														config.to_yaml_file(lowerCAmelCase_      )
							print(F"accelerate configuration saved at {config_file}"      )
def       snake_case_		(     ):
							__lowercase      :							str				=     config_command_parser()
							__lowercase      :							str				=     parser.parse_args()
							config_command(lowerCAmelCase_      )


if __name__ == "__main__":
	main()