






import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 		lowerCAmelCase     ( UpperCAmelCase_      ,       unittest.TestCase			):

					'''simple docstring'''

					_A						:			List[Any]									=      RobertaTokenizer
					_A						:			Optional[int]									=      RobertaTokenizerFast
					_A						:			Union[str, Any]									=      True
					_A						:			Union[str, Any]									=      {"cls_token": "<s>"}



					def       lowerCAmelCase				(  self				:	str							)			->  Tuple:


												"""simple docstring"""

												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												__lowercase      :							List[Any]				=     [
												    """l""",
												    """o""",
												    """w""",
												    """e""",
												    """r""",
												    """s""",
												    """t""",
												    """i""",
												    """d""",
												    """n""",
												    """\u0120""",
												    """\u0120l""",
												    """\u0120n""",
												    """\u0120lo""",
												    """\u0120low""",
												    """er""",
												    """\u0120lowest""",
												    """\u0120newer""",
												    """\u0120wider""",
												    """<unk>""",
												]
												__lowercase      :							Union[str, Any]				=     dict(zip(_snake_case		,     range(len(_snake_case							)							)							)							)
												__lowercase      :							int				=     ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
												__lowercase      :							Any				=     {"""unk_token""": """<unk>"""}

												__lowercase      :							str				=     os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["""vocab_file"""]							)
												__lowercase      :							List[Any]				=     os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["""merges_file"""]							)
												with open(self.vocab_file		,     """w"""		,     encoding="""utf-8"""							) as fp:
																			fp.write(json.dumps(_snake_case							) + """\n"""							)
												with open(self.merges_file		,     """w"""		,     encoding="""utf-8"""							) as fp:
																			fp.write("""\n""".join(_snake_case							)							)



					def       lowerCAmelCase				(  self				:	Optional[Any]		,     **__a				:	str							)			->  int:


												"""simple docstring"""

												kwargs.update(self.special_tokens_map							)
												return self.tokenizer_class.from_pretrained(self.tmpdirname		,     **_snake_case							)



					def       lowerCAmelCase				(  self				:	Union[str, Any]		,     **__a				:	Dict							)			->  str:


												"""simple docstring"""

												kwargs.update(self.special_tokens_map							)
												return RobertaTokenizerFast.from_pretrained(self.tmpdirname		,     **_snake_case							)



					def       lowerCAmelCase				(  self				:	Optional[Any]		,     __a				:	Optional[int]							)			->  List[str]:


												"""simple docstring"""

												__lowercase      :							Any				=     """lower newer"""
												__lowercase      :							List[Any]				=     """lower newer"""
												return input_text, output_text



					def       lowerCAmelCase				(  self				:	Union[str, Any]							)			->  Optional[Any]:


												"""simple docstring"""

												__lowercase      :							Tuple				=     self.tokenizer_class(self.vocab_file		,     self.merges_file		,     **self.special_tokens_map							)
												__lowercase      :							Optional[int]				=     """lower newer"""
												__lowercase      :							Union[str, Any]				=     ["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
												__lowercase      :							Any				=     tokenizer.tokenize(_snake_case							)  # , add_prefix_space=True)
												self.assertListEqual(_snake_case		,     _snake_case							)

												__lowercase      :							Optional[Any]				=     tokens + [tokenizer.unk_token]
												__lowercase      :							Optional[Any]				=     [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case							)		,     _snake_case							)



					def       lowerCAmelCase				(  self				:	List[Any]							)			->  Dict:


												"""simple docstring"""

												__lowercase      :							Tuple				=     self.get_tokenizer()

												self.assertListEqual(tokenizer.encode("""Hello world!"""		,     add_special_tokens=_snake_case							)		,     [0, 31414, 232, 328, 2]							)
												self.assertListEqual(
												    tokenizer.encode("""Hello world! cécé herlolip 418"""		,     add_special_tokens=_snake_case							)		,     [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]		,     )



					@slow
					def       lowerCAmelCase				(  self				:	Any							)			->  Union[str, Any]:


												"""simple docstring"""

												__lowercase      :							Optional[int]				=     self.tokenizer_class.from_pretrained("""roberta-base"""							)

												__lowercase      :							Optional[int]				=     tokenizer.encode("""sequence builders"""		,     add_special_tokens=_snake_case							)
												__lowercase      :							Optional[int]				=     tokenizer.encode("""multi-sequence build"""		,     add_special_tokens=_snake_case							)

												__lowercase      :							Optional[int]				=     tokenizer.encode(
												    """sequence builders"""		,     add_special_tokens=_snake_case		,     add_prefix_space=_snake_case							)
												__lowercase      :							Dict				=     tokenizer.encode(
												    """sequence builders"""		,     """multi-sequence build"""		,     add_special_tokens=_snake_case		,     add_prefix_space=_snake_case							)

												__lowercase      :							Optional[int]				=     tokenizer.build_inputs_with_special_tokens(_snake_case							)
												__lowercase      :							Union[str, Any]				=     tokenizer.build_inputs_with_special_tokens(_snake_case		,     _snake_case							)

												assert encoded_sentence == encoded_text_from_decode
												assert encoded_pair == encoded_pair_from_decode



					def       lowerCAmelCase				(  self				:	Union[str, Any]							)			->  Tuple:


												"""simple docstring"""

												__lowercase      :							Tuple				=     self.get_tokenizer()

												__lowercase      :							Optional[int]				=     """Encode this sequence."""
												__lowercase      :							Union[str, Any]				=     tokenizer.byte_encoder[""" """.encode("""utf-8"""							)[0]]

												# Testing encoder arguments
												__lowercase      :							Optional[int]				=     tokenizer.encode(_snake_case		,     add_special_tokens=_snake_case		,     add_prefix_space=_snake_case							)
												__lowercase      :							Tuple				=     tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
												self.assertNotEqual(_snake_case		,     _snake_case							)

												__lowercase      :							List[Any]				=     tokenizer.encode(_snake_case		,     add_special_tokens=_snake_case		,     add_prefix_space=_snake_case							)
												__lowercase      :							int				=     tokenizer.convert_ids_to_tokens(encoded[0]							)[0]
												self.assertEqual(_snake_case		,     _snake_case							)

												tokenizer.add_special_tokens({"""bos_token""": """<s>"""}							)
												__lowercase      :							Optional[int]				=     tokenizer.encode(_snake_case		,     add_special_tokens=_snake_case							)
												__lowercase      :							List[Any]				=     tokenizer.convert_ids_to_tokens(encoded[1]							)[0]
												self.assertNotEqual(_snake_case		,     _snake_case							)

												# Testing spaces after special tokens
												__lowercase      :							Tuple				=     """<mask>"""
												tokenizer.add_special_tokens(
												    {"""mask_token""": AddedToken(_snake_case		,     lstrip=_snake_case		,     rstrip=_snake_case							)}							)  # mask token has a left space
												__lowercase      :							Any				=     tokenizer.convert_tokens_to_ids(_snake_case							)

												__lowercase      :							Optional[Any]				=     """Encode <mask> sequence"""
												__lowercase      :							Tuple				=     """Encode <mask>sequence"""

												__lowercase      :							List[Any]				=     tokenizer.encode(_snake_case							)
												__lowercase      :							Any				=     encoded.index(_snake_case							)
												__lowercase      :							List[Any]				=     tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
												self.assertEqual(_snake_case		,     _snake_case							)

												__lowercase      :							Any				=     tokenizer.encode(_snake_case							)
												__lowercase      :							Dict				=     encoded.index(_snake_case							)
												__lowercase      :							Union[str, Any]				=     tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]							)[0]
												self.assertNotEqual(_snake_case		,     _snake_case							)



					def       lowerCAmelCase				(  self				:	Tuple							)			->  str:


												"""simple docstring"""

												pass



					def       lowerCAmelCase				(  self				:	int							)			->  Union[str, Any]:


												"""simple docstring"""

												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
																										__lowercase      :							str				=     self.rust_tokenizer_class.from_pretrained(_snake_case		,     **_snake_case							)
																										__lowercase      :							List[str]				=     self.tokenizer_class.from_pretrained(_snake_case		,     **_snake_case							)
																										__lowercase      :							str				=     """A, <mask> AllenNLP sentence."""
																										__lowercase      :							Any				=     tokenizer_r.encode_plus(_snake_case		,     add_special_tokens=_snake_case		,     return_token_type_ids=_snake_case							)
																										__lowercase      :							List[str]				=     tokenizer_p.encode_plus(_snake_case		,     add_special_tokens=_snake_case		,     return_token_type_ids=_snake_case							)

																										# token_type_ids should put 0 everywhere
																										self.assertEqual(sum(tokens_r["""token_type_ids"""]							)		,     sum(tokens_p["""token_type_ids"""]							)							)

																										# attention_mask should put 1 everywhere, so sum over length should be 1
																										self.assertEqual(
																										    sum(tokens_r["""attention_mask"""]							) / len(tokens_r["""attention_mask"""]							)		,     sum(tokens_p["""attention_mask"""]							) / len(tokens_p["""attention_mask"""]							)		,     )

																										__lowercase      :							Union[str, Any]				=     tokenizer_r.convert_ids_to_tokens(tokens_r["""input_ids"""]							)
																										__lowercase      :							Union[str, Any]				=     tokenizer_p.convert_ids_to_tokens(tokens_p["""input_ids"""]							)

																										# Rust correctly handles the space before the mask while python doesnt
																										self.assertSequenceEqual(tokens_p["""input_ids"""]		,     [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]							)
																										self.assertSequenceEqual(tokens_r["""input_ids"""]		,     [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]							)

																										self.assertSequenceEqual(
																										    _snake_case		,     ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)
																										self.assertSequenceEqual(
																										    _snake_case		,     ["""<s>""", """A""", """,""", """<mask>""", """ĠAllen""", """N""", """LP""", """Ġsentence""", """.""", """</s>"""]							)



					def       lowerCAmelCase				(  self				:	Union[str, Any]							)			->  Any:


												"""simple docstring"""

												for trim_offsets, add_prefix_space in itertools.product([True, False]		,     repeat=2							):
																			__lowercase      :							Any				=     self.rust_tokenizer_class.from_pretrained(
																			    self.tmpdirname		,     use_fast=_snake_case		,     add_prefix_space=_snake_case		,     trim_offsets=_snake_case							)

																			__lowercase      :							str				=     json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()							)
																			__lowercase      :							str				=     json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()							)

																			self.assertEqual(pre_tokenizer_state["""add_prefix_space"""]		,     _snake_case							)

																			self.assertEqual(post_processor_state["""add_prefix_space"""]		,     _snake_case							)
																			self.assertEqual(post_processor_state["""trim_offsets"""]		,     _snake_case							)



					def       lowerCAmelCase				(  self				:	Tuple							)			->  Optional[int]:


												"""simple docstring"""

												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"							):
																										__lowercase      :							str				=     """hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
																										__lowercase      :							Optional[Any]				=     F"{text_of_1_token} {text_of_1_token}"

																										__lowercase      :							Any				=     self.rust_tokenizer_class.from_pretrained(
																										    _snake_case		,     use_fast=_snake_case		,     add_prefix_space=_snake_case		,     trim_offsets=_snake_case							)
																										__lowercase      :							Tuple				=     tokenizer_r(_snake_case		,     return_offsets_mapping=_snake_case		,     add_special_tokens=_snake_case							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(_snake_case							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(_snake_case							) + 1, len(_snake_case							) + 1 + len(_snake_case							))		,     )

																										__lowercase      :							List[str]				=     self.rust_tokenizer_class.from_pretrained(
																										    _snake_case		,     use_fast=_snake_case		,     add_prefix_space=_snake_case		,     trim_offsets=_snake_case							)
																										__lowercase      :							str				=     tokenizer_r(_snake_case		,     return_offsets_mapping=_snake_case		,     add_special_tokens=_snake_case							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(_snake_case							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(_snake_case							) + 1, len(_snake_case							) + 1 + len(_snake_case							))		,     )

																										__lowercase      :							Dict				=     self.rust_tokenizer_class.from_pretrained(
																										    _snake_case		,     use_fast=_snake_case		,     add_prefix_space=_snake_case		,     trim_offsets=_snake_case							)
																										__lowercase      :							Tuple				=     tokenizer_r(_snake_case		,     return_offsets_mapping=_snake_case		,     add_special_tokens=_snake_case							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(_snake_case							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(_snake_case							), len(_snake_case							) + 1 + len(_snake_case							))		,     )

																										__lowercase      :							int				=     self.rust_tokenizer_class.from_pretrained(
																										    _snake_case		,     use_fast=_snake_case		,     add_prefix_space=_snake_case		,     trim_offsets=_snake_case							)
																										__lowercase      :							str				=     tokenizer_r(_snake_case		,     return_offsets_mapping=_snake_case		,     add_special_tokens=_snake_case							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, len(_snake_case							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (len(_snake_case							), len(_snake_case							) + 1 + len(_snake_case							))		,     )

																										__lowercase      :							Any				=     F" {text}"

																										# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																										#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																										# )
																										# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																										# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																										# self.assertEqual(
																										#     encoding.offset_mapping[1],
																										#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																										# )

																										__lowercase      :							Optional[Any]				=     self.rust_tokenizer_class.from_pretrained(
																										    _snake_case		,     use_fast=_snake_case		,     add_prefix_space=_snake_case		,     trim_offsets=_snake_case							)
																										__lowercase      :							Any				=     tokenizer_r(_snake_case		,     return_offsets_mapping=_snake_case		,     add_special_tokens=_snake_case							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (1, 1 + len(_snake_case							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(_snake_case							) + 1, 1 + len(_snake_case							) + 1 + len(_snake_case							))		,     )

																										__lowercase      :							Optional[Any]				=     self.rust_tokenizer_class.from_pretrained(
																										    _snake_case		,     use_fast=_snake_case		,     add_prefix_space=_snake_case		,     trim_offsets=_snake_case							)
																										__lowercase      :							Dict				=     tokenizer_r(_snake_case		,     return_offsets_mapping=_snake_case		,     add_special_tokens=_snake_case							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, 1 + len(_snake_case							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(_snake_case							), 1 + len(_snake_case							) + 1 + len(_snake_case							))		,     )

																										__lowercase      :							Any				=     self.rust_tokenizer_class.from_pretrained(
																										    _snake_case		,     use_fast=_snake_case		,     add_prefix_space=_snake_case		,     trim_offsets=_snake_case							)
																										__lowercase      :							Tuple				=     tokenizer_r(_snake_case		,     return_offsets_mapping=_snake_case		,     add_special_tokens=_snake_case							)
																										self.assertEqual(encoding.offset_mapping[0]		,     (0, 1 + len(_snake_case							))							)
																										self.assertEqual(
																										    encoding.offset_mapping[1]		,     (1 + len(_snake_case							), 1 + len(_snake_case							) + 1 + len(_snake_case							))		,     )







# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


lowerCamelCase : int						   =       '''Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine'''
def       snake_case_		(     ):
							__lowercase      :							List[Any]				=     _ask_options(
							    """In which compute environment are you running?"""       ,       ["""This machine""", """AWS (Amazon SageMaker)"""]       ,       _convert_compute_environment       ,       )
							if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
														__lowercase      :							Union[str, Any]				=     get_sagemaker_input()
							else:
														__lowercase      :							str				=     get_cluster_input()
							return config
def       snake_case_		(     lowerCAmelCase_       :						List[str]=None      ):
							if subparsers is not None:
														__lowercase      :							Optional[int]				=     subparsers.add_parser("""config"""       ,       description=lowerCAmelCase_      )
							else:
														__lowercase      :							List[str]				=     argparse.ArgumentParser("""Accelerate config command"""       ,       description=lowerCAmelCase_      )

							parser.add_argument(
							    """--config_file"""       ,       default=lowerCAmelCase_       ,       help=(
							        """The path to use to store the config file. Will default to a file named default_config.yaml in the cache """
							        """location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have """
							        """such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed """
							        """with 'huggingface'."""
							    )       ,       )

							if subparsers is not None:
														parser.set_defaults(func=lowerCAmelCase_      )
							return parser
def       snake_case_		(     lowerCAmelCase_       :						Tuple      ):
							__lowercase      :							Union[str, Any]				=     get_user_input()
							if args.config_file is not None:
														__lowercase      :							List[Any]				=     args.config_file
							else:
														if not os.path.isdir(lowerCAmelCase_      ):
																					os.makedirs(lowerCAmelCase_      )
														__lowercase      :							Any				=     default_yaml_config_file

							if config_file.endswith(""".json"""      ):
														config.to_json_file(lowerCAmelCase_      )
							else:
														config.to_yaml_file(lowerCAmelCase_      )
							print(F"accelerate configuration saved at {config_file}"      )
def       snake_case_		(     ):
							__lowercase      :							str				=     config_command_parser()
							__lowercase      :							str				=     parser.parse_args()
							config_command(lowerCAmelCase_      )


if __name__ == "__main__":
	main()