


import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class        UpperCamelCase__   (	__snake_case							,				unittest.TestCase   ):
					__SCREAMING_SNAKE_CASE       :  List[Any]						=   MgpstrTokenizer
					__SCREAMING_SNAKE_CASE       :  str						=   False
					__SCREAMING_SNAKE_CASE       :  str						=   {}
					__SCREAMING_SNAKE_CASE       :  Union[str, Any]						=   False

					def   UpperCAmelCase__       (     self       :   Optional[Any]       ):






										'''simple docstring'''






										super().setUp()

										# fmt: off
										lowercase_  							=			["""[GO]""", """[s]""", """0""", """1""", """2""", """3""", """4""", """5""", """6""", """7""", """8""", """9""", """a""", """b""", """c""", """d""", """e""", """f""", """g""", """h""", """i""", """j""", """k""", """l""", """m""", """n""", """o""", """p""", """q""", """r""", """s""", """t""", """u""", """v""", """w""", """x""", """y""", """z"""]
										# fmt: on
										lowercase_  							=			dict(zip(__UpperCamelCase							,					range(len(__UpperCamelCase       )       )       )       )

										lowercase_  							=			os.path.join(self.tmpdirname							,					VOCAB_FILES_NAMES["""vocab_file"""]       )
										with open(self.vocab_file							,					"""w"""							,					encoding="""utf-8"""       ) as fp:
															fp.write(json.dumps(__UpperCamelCase       ) + """\n"""       )

					def   UpperCAmelCase__       (     self       :   int							,					**UpperCamelCase__       :   int       ):






										'''simple docstring'''






										return MgpstrTokenizer.from_pretrained(self.tmpdirname							,					**__UpperCamelCase       )

					def   UpperCAmelCase__       (     self       :   Any							,					UpperCamelCase__       :   Dict       ):






										'''simple docstring'''






										lowercase_  							=			"""tester"""
										lowercase_  							=			"""tester"""
										return input_text, output_text

					@unittest.skip("""MGP-STR always lower cases letters."""       )
					def   UpperCAmelCase__       (     self       :   str       ):






										'''simple docstring'''






										pass

					def   UpperCAmelCase__       (     self       :   Union[str, Any]       ):






										'''simple docstring'''






										lowercase_  							=			self.get_tokenizers(do_lower_case=__UpperCamelCase       )
										for tokenizer in tokenizers:
															with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																				lowercase_  							=			"""[SPECIAL_TOKEN]"""

																				tokenizer.add_special_tokens({"""cls_token""": special_token}       )
																				lowercase_  							=			tokenizer.encode([special_token]							,					add_special_tokens=__UpperCamelCase       )
																				self.assertEqual(len(__UpperCamelCase       )							,					1       )

																				lowercase_  							=			tokenizer.decode(__UpperCamelCase							,					skip_special_tokens=__UpperCamelCase       )
																				self.assertTrue(special_token not in decoded       )

					def   UpperCAmelCase__       (     self       :   Optional[Any]       ):






										'''simple docstring'''






										lowercase_  							=			self.get_tokenizers()
										for tokenizer in tokenizers:
															with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
																				lowercase_							,					lowercase_  							=			self.get_input_output_texts(__UpperCamelCase       )

																				lowercase_  							=			tokenizer.tokenize(__UpperCamelCase       )
																				lowercase_  							=			tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
																				lowercase_  							=			tokenizer.encode(__UpperCamelCase							,					add_special_tokens=__UpperCamelCase       )
																				self.assertListEqual(__UpperCamelCase							,					__UpperCamelCase       )

																				lowercase_  							=			tokenizer.convert_ids_to_tokens(__UpperCamelCase       )
																				self.assertNotEqual(len(__UpperCamelCase       )							,					0       )
																				lowercase_  							=			tokenizer.decode(__UpperCamelCase       )
																				self.assertIsInstance(__UpperCamelCase							,					__UpperCamelCase       )

																				self.assertEqual(text_a.replace(""" """							,					""""""       )							,					__UpperCamelCase       )

					@unittest.skip("""MGP-STR tokenizer only handles one sequence."""       )
					def   UpperCAmelCase__       (     self       :   List[Any]       ):






										'''simple docstring'''






										pass







					@unittest.skip("""inputs cannot be pretokenized in MgpstrTokenizer"""       )
					def   UpperCAmelCase__       (     self       :   Tuple       ):






										'''simple docstring'''






										pass



def 			UpperCAmelCase_				(      UpperCAmelCase__  ):
					if not isinstance(UpperCAmelCase__						,							UpperCAmelCase__  ):
										raise TypeError("""Input value must be an 'int' type"""  )

					lowercase_  							=			0
					while number:
										position += 1
										number >>= 1

					return position


if __name__ == "__main__":
							import doctest

							doctest.testmod()


