



import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			_lowerCAmelCase   (							_UpperCAmelCase    ,					unittest.TestCase			):






			"""simple docstring"""





			lowercase__      :  Union[str, Any]      =			GPTaTokenizer
			lowercase__      :  str      =			GPTaTokenizerFast
			lowercase__      :  List[Any]      =			True
			lowercase__      :  List[str]      =			{"""add_prefix_space""": True}
			lowercase__      :  Optional[Any]      =			False





			def 		snake_case__			(				self	:       int   )						->					Optional[Any]:
									"""simple docstring"""







									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									__lowercase					=				[
									    """l""",
									    """o""",
									    """w""",
									    """e""",
									    """r""",
									    """s""",
									    """t""",
									    """i""",
									    """d""",
									    """n""",
									    """\u0120""",
									    """\u0120l""",
									    """\u0120n""",
									    """\u0120lo""",
									    """\u0120low""",
									    """er""",
									    """\u0120lowest""",
									    """\u0120newer""",
									    """\u0120wider""",
									    """<unk>""",
									    """<|endoftext|>""",
									]
									__lowercase					=				dict(zip(lowercase			,     range(len(lowercase   )   )   )   )
									__lowercase					=				["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
									__lowercase					=				{"""unk_token""": """<unk>"""}

									__lowercase					=				os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["""vocab_file"""]   )
									__lowercase					=				os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["""merges_file"""]   )
									with open(self.vocab_file			,     """w"""			,     encoding="""utf-8"""   ) as fp:
															fp.write(json.dumps(lowercase   ) + """\n"""   )
									with open(self.merges_file			,     """w"""			,     encoding="""utf-8"""   ) as fp:
															fp.write("""\n""".join(lowercase   )   )





			def 		snake_case__			(				self	:       Union[str, Any]			,     **lowercase	:       Tuple   )						->					Optional[Any]:
									"""simple docstring"""







									kwargs.update(self.special_tokens_map   )
									return GPTaTokenizer.from_pretrained(self.tmpdirname			,     **lowercase   )





			def 		snake_case__			(				self	:       Dict			,     **lowercase	:       Optional[int]   )						->					str:
									"""simple docstring"""







									kwargs.update(self.special_tokens_map   )
									return GPTaTokenizerFast.from_pretrained(self.tmpdirname			,     **lowercase   )





			def 		snake_case__			(				self	:       List[str]			,     lowercase	:       List[Any]   )						->					Optional[int]:
									"""simple docstring"""







									__lowercase					=				"""lower newer"""
									__lowercase					=				"""lower newer"""
									return input_text, output_text





			def 		snake_case__			(				self	:       Union[str, Any]   )						->					Any:
									"""simple docstring"""







									__lowercase					=				GPTaTokenizer(self.vocab_file			,     self.merges_file			,     **self.special_tokens_map   )
									__lowercase					=				"""lower newer"""
									__lowercase					=				["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
									__lowercase					=				tokenizer.tokenize(lowercase			,     add_prefix_space=lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )

									__lowercase					=				tokens + [tokenizer.unk_token]
									__lowercase					=				[14, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase   )			,     lowercase   )





			def 		snake_case__			(				self	:       Any   )						->					str:
									"""simple docstring"""







									if not self.test_rust_tokenizer:
															return

									__lowercase					=				self.get_tokenizer()
									__lowercase					=				self.get_rust_tokenizer(add_prefix_space=lowercase   )

									__lowercase					=				"""lower newer"""

									# Testing tokenization
									__lowercase					=				tokenizer.tokenize(lowercase			,     add_prefix_space=lowercase   )
									__lowercase					=				rust_tokenizer.tokenize(lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )

									# Testing conversion to ids without special tokens
									__lowercase					=				tokenizer.encode(lowercase			,     add_special_tokens=lowercase			,     add_prefix_space=lowercase   )
									__lowercase					=				rust_tokenizer.encode(lowercase			,     add_special_tokens=lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )

									# Testing conversion to ids with special tokens
									__lowercase					=				self.get_rust_tokenizer(add_prefix_space=lowercase   )
									__lowercase					=				tokenizer.encode(lowercase			,     add_prefix_space=lowercase   )
									__lowercase					=				rust_tokenizer.encode(lowercase   )
									self.assertListEqual(lowercase			,     lowercase   )

									# Testing the unknown token
									__lowercase					=				tokens + [rust_tokenizer.unk_token]
									__lowercase					=				[14, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowercase   )			,     lowercase   )





			def 		snake_case__			(				self	:       List[str]			,     *lowercase	:       Tuple			,     **lowercase	:       int   )						->					List[str]:
									"""simple docstring"""







									pass





			def 		snake_case__			(				self	:       Tuple			,     lowercase	:       Optional[Any]=15   )						->					Union[str, Any]:
									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
																					__lowercase					=				self.rust_tokenizer_class.from_pretrained(lowercase			,     **lowercase   )

																					# Simple input
																					__lowercase					=				"""This is a simple input"""
																					__lowercase					=				["""This is a simple input 1""", """This is a simple input 2"""]
																					__lowercase					=				("""This is a simple input""", """This is a pair""")
																					__lowercase					=				[
																					    ("""This is a simple input 1""", """This is a simple input 2"""),
																					    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																					]

																					# Simple input tests
																					self.assertRaises(lowercase			,     tokenizer_r.encode			,     lowercase			,     max_length=lowercase			,     padding="""max_length"""   )

																					# Simple input
																					self.assertRaises(lowercase			,     tokenizer_r.encode_plus			,     lowercase			,     max_length=lowercase			,     padding="""max_length"""   )

																					# Simple input
																					self.assertRaises(
																					    lowercase			,     tokenizer_r.batch_encode_plus			,     lowercase			,     max_length=lowercase			,     padding="""max_length"""			,     )

																					# Pair input
																					self.assertRaises(lowercase			,     tokenizer_r.encode			,     lowercase			,     max_length=lowercase			,     padding="""max_length"""   )

																					# Pair input
																					self.assertRaises(lowercase			,     tokenizer_r.encode_plus			,     lowercase			,     max_length=lowercase			,     padding="""max_length"""   )

																					# Pair input
																					self.assertRaises(
																					    lowercase			,     tokenizer_r.batch_encode_plus			,     lowercase			,     max_length=lowercase			,     padding="""max_length"""			,     )





			def 		snake_case__			(				self	:       Any   )						->					List[str]:
									"""simple docstring"""







									__lowercase					=				GPTaTokenizer.from_pretrained(self.tmpdirname			,     pad_token="""<pad>"""   )

									# Simple input
									__lowercase					=				"""This is a simple input"""
									__lowercase					=				["""This is a simple input looooooooong""", """This is a simple input"""]
									__lowercase					=				("""This is a simple input""", """This is a pair""")
									__lowercase					=				[
									    ("""This is a simple input loooooong""", """This is a simple input"""),
									    ("""This is a simple pair loooooong""", """This is a simple pair"""),
									]

									__lowercase					=				tokenizer.pad_token_id

									__lowercase					=				tokenizer(lowercase			,     padding="""max_length"""			,     max_length=30			,     return_tensors="""np"""   )
									__lowercase					=				tokenizer(lowercase			,     padding=lowercase			,     truncate=lowercase			,     return_tensors="""np"""   )
									__lowercase					=				tokenizer(*lowercase			,     padding="""max_length"""			,     max_length=60			,     return_tensors="""np"""   )
									__lowercase					=				tokenizer(lowercase			,     padding=lowercase			,     truncate=lowercase			,     return_tensors="""np"""   )

									# s
									# test single string max_length padding
									self.assertEqual(out_s["""input_ids"""].shape[-1]			,     30   )
									self.assertTrue(pad_token_id in out_s["""input_ids"""]   )
									self.assertTrue(0 in out_s["""attention_mask"""]   )

									# s2
									# test automatic padding
									self.assertEqual(out_sa["""input_ids"""].shape[-1]			,     33   )
									# long slice doesn't have padding
									self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]   )
									self.assertFalse(0 in out_sa["""attention_mask"""][0]   )
									# short slice does have padding
									self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]   )
									self.assertTrue(0 in out_sa["""attention_mask"""][1]   )

									# p
									# test single pair max_length padding
									self.assertEqual(out_p["""input_ids"""].shape[-1]			,     60   )
									self.assertTrue(pad_token_id in out_p["""input_ids"""]   )
									self.assertTrue(0 in out_p["""attention_mask"""]   )

									# p2
									# test automatic padding pair
									self.assertEqual(out_pa["""input_ids"""].shape[-1]			,     52   )
									# long slice pair doesn't have padding
									self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]   )
									self.assertFalse(0 in out_pa["""attention_mask"""][0]   )
									# short slice pair does have padding
									self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]   )
									self.assertTrue(0 in out_pa["""attention_mask"""][1]   )





			def 		snake_case__			(				self	:       Any   )						->					Optional[Any]:
									"""simple docstring"""







									__lowercase					=				"""$$$"""
									__lowercase					=				GPTaTokenizer.from_pretrained(self.tmpdirname			,     bos_token=lowercase			,     add_bos_token=lowercase   )

									__lowercase					=				"""This is a simple input"""
									__lowercase					=				["""This is a simple input 1""", """This is a simple input 2"""]

									__lowercase					=				tokenizer.bos_token_id

									__lowercase					=				tokenizer(lowercase   )
									__lowercase					=				tokenizer(lowercase   )

									self.assertEqual(out_s.input_ids[0]			,     lowercase   )
									self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids   )   )

									__lowercase					=				tokenizer.decode(out_s.input_ids   )
									__lowercase					=				tokenizer.batch_decode(out_sa.input_ids   )

									self.assertEqual(decode_s.split()[0]			,     lowercase   )
									self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa   )   )





			def 		snake_case__			(				self	:       Union[str, Any]   )						->					List[Any]:
									"""simple docstring"""







									pass







			def 		snake_case__			(				self	:       Union[str, Any]   )						->					Union[str, Any]:
									"""simple docstring"""







									__lowercase					=				[self.get_tokenizer(do_lower_case=lowercase			,     add_bos_token=lowercase   )]
									for tokenizer in tokenizers:
															with self.subTest(F"{tokenizer.__class__.__name__}"   ):
																					__lowercase					=				"""Encode this."""
																					__lowercase					=				"""This one too please."""
																					__lowercase					=				tokenizer.encode(lowercase			,     add_special_tokens=lowercase   )
																					encoded_sequence += tokenizer.encode(lowercase			,     add_special_tokens=lowercase   )
																					__lowercase					=				tokenizer.encode_plus(
																					    lowercase			,     lowercase			,     add_special_tokens=lowercase			,     return_special_tokens_mask=lowercase			,     )
																					__lowercase					=				encoded_sequence_dict["""input_ids"""]
																					__lowercase					=				encoded_sequence_dict["""special_tokens_mask"""]
																					self.assertEqual(len(lowercase   )			,     len(lowercase   )   )

																					__lowercase					=				[
																					    (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowercase   )
																					]
																					__lowercase					=				[x for x in filtered_sequence if x is not None]
																					self.assertEqual(lowercase			,     lowercase   )





@require_tokenizers
class 			_lowerCAmelCase   (							unittest.TestCase			):






			"""simple docstring"""





			def 		snake_case__			(				self	:       int   )						->					Union[str, Any]:
									"""simple docstring"""







									__lowercase					=				AutoTokenizer.from_pretrained("""facebook/opt-350m"""			,     from_slow=lowercase   )
									__lowercase					=				"""A photo of a cat"""

									__lowercase					=				tokenizer.encode(
									    lowercase			,     )
									self.assertEqual(lowercase			,     [2, 250, 1_345, 9, 10, 4_758]   )
									tokenizer.save_pretrained("""test_opt"""   )

									__lowercase					=				AutoTokenizer.from_pretrained("""./test_opt"""   )
									__lowercase					=				tokenizer.encode(
									    lowercase			,     )
									self.assertEqual(lowercase			,     [2, 250, 1_345, 9, 10, 4_758]   )





			def 		snake_case__			(				self	:       str   )						->					List[str]:
									"""simple docstring"""







									__lowercase					=				AutoTokenizer.from_pretrained("""facebook/opt-350m"""			,     use_slow=lowercase   )
									__lowercase					=				"""A photo of a cat"""

									__lowercase					=				tokenizer.encode(
									    lowercase			,     )
									# Same as above
									self.assertEqual(lowercase			,     [2, 250, 1_345, 9, 10, 4_758]   )







			@unittest.skip("""This test is failing because of a bug in the fast tokenizer"""   )
			def 		snake_case__			(				self	:       Any   )						->					List[str]:
									"""simple docstring"""







									__lowercase					=				AutoTokenizer.from_pretrained("""facebook/opt-350m"""			,     from_slow=lowercase   )

									__lowercase					=				"""bos"""
									__lowercase					=				tokenizer.get_vocab()["""bos"""]

									__lowercase					=				"""A photo of a cat"""
									__lowercase					=				tokenizer.encode(
									    lowercase			,     )
									# We changed the bos token
									self.assertEqual(lowercase			,     [31_957, 250, 1_345, 9, 10, 4_758]   )
									tokenizer.save_pretrained("""./tok"""   )
									__lowercase					=				AutoTokenizer.from_pretrained("""./tok"""   )
									self.assertTrue(tokenizer.is_fast   )
									__lowercase					=				tokenizer.encode(
									    lowercase			,     )
									self.assertEqual(lowercase			,     [31_957, 250, 1_345, 9, 10, 4_758]   )










import random






def  UpperCAmelCase__    (    lowercase__						,       lowercase__						,       lowercase__ = False   ) ->      dict:
						__lowercase					=				{i: [] for i in range(lowercase__   )}

						# if probability is greater or equal than 1, then generate a complete graph
						if probability >= 1:
												return complete_graph(lowercase__   )
						# if probability is lower or equal than 0, then return a graph without edges
						if probability <= 0:
												return graph

						# for each couple of nodes, add an edge from u to v
						# if the number randomly generated is greater than probability probability
						for i in range(lowercase__   ):
												for j in range(i + 1						,       lowercase__   ):
																		if random.random() < probability:
																								graph[i].append(lowercase__   )
																								if not directed:
																														# if the graph is undirected, add an edge in from j to i, either
																														graph[j].append(lowercase__   )
						return graph






def  UpperCAmelCase__    (    lowercase__   ) ->      dict:
						return {
						    i: [j for j in range(lowercase__   ) if i != j] for i in range(lowercase__   )
						}


if __name__ == "__main__":
			import doctest

			doctest.testmod()





