



'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


a     :    Union[str, Any]      						=					logging.get_logger(__name__)

a     :    Dict      						=					{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a     :    List[str]      						=					{
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

a     :    Optional[Any]      						=					{
    """junnyu/roformer_chinese_small""": 1_5_3_6,
    """junnyu/roformer_chinese_base""": 1_5_3_6,
    """junnyu/roformer_chinese_char_small""": 5_1_2,
    """junnyu/roformer_chinese_char_base""": 5_1_2,
    """junnyu/roformer_small_discriminator""": 1_2_8,
    """junnyu/roformer_small_generator""": 1_2_8,
}


a     :    int      						=					{
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}

class     UpperCamelCase_				(							__magic_name__    ):
	lowercase    =				VOCAB_FILES_NAMES
	lowercase    =				PRETRAINED_VOCAB_FILES_MAP
	lowercase    =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase    =				PRETRAINED_INIT_CONFIGURATION
	lowercase    =				RoFormerTokenizer





	def __init__(						self	,	A=None	,	A=None	,	A=True	,	A="[UNK]"	,	A="[SEP]"	,	A="[PAD]"	,	A="[CLS]"	,	A="[MASK]"	,	A=True	,	A=None	,	**A	,	)    ->    Tuple:
								super().__init__(
								    A	,	tokenizer_file=A	,	do_lower_case=A	,	unk_token=A	,	sep_token=A	,	pad_token=A	,	cls_token=A	,	mask_token=A	,	tokenize_chinese_chars=A	,	strip_accents=A	,	**A	,	)

								UpperCAmelCase  :	Any							  =	json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
								if (
								    pre_tok_state.get("""lowercase"""	,	A					) != do_lower_case
								    or pre_tok_state.get("""strip_accents"""	,	A					) != strip_accents
								):
															UpperCAmelCase  :	Any							  =	getattr(A	,	pre_tok_state.pop("""type"""					)					)
															UpperCAmelCase  :	str							  =	do_lower_case
															UpperCAmelCase  :	Optional[Any]							  =	strip_accents
															UpperCAmelCase  :	Optional[int]							  =	pre_tok_class(**A					)

								UpperCAmelCase  :	str							  =	do_lower_case





	def __getstate__(						self					)    ->    Any:
								UpperCAmelCase  :	List[Any]							  =	self.__dict__.copy()
								UpperCAmelCase  :	Tuple							  =	BertPreTokenizer()
								return state





	def __setstate__(						self	,	A					)    ->    List[Any]:
								UpperCAmelCase  :	int							  =	d
								UpperCAmelCase  :	List[str]							  =	self.__dict__["""_tokenizer"""].get_vocab()
								UpperCAmelCase  :	Optional[int]							  =	PreTokenizer.custom(JiebaPreTokenizer(A					)					)





	def 				_lowercase(						self	,	A	,	A=None					)    ->    Tuple:
								UpperCAmelCase  :	List[Any]							  =	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

								return output





	def 				_lowercase(						self	,	A	,	A = None					)    ->    List[int]:
								UpperCAmelCase  :	Optional[Any]							  =	[self.sep_token_id]
								UpperCAmelCase  :	Dict							  =	[self.cls_token_id]
								if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
								return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]





	def 				_lowercase(						self	,	A	,	A = None					)    ->    Tuple[str]:
								UpperCAmelCase  :	Dict							  =	self._tokenizer.model.save(A	,	name=A					)
								return tuple(A					)






	def 				_lowercase(						self	,	A	,	A=None	,	A=None	,	A=False	,	**A	,	)    ->    List[Any]:
								UpperCAmelCase  :	Optional[int]							  =	BertPreTokenizer()
								return super().save_pretrained(A	,	A	,	A	,	A	,	**A					)






'''simple docstring'''




import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a     :    List[Any]      						=					get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")

@require_sentencepiece
@require_tokenizers
class     UpperCamelCase_				(							__magic_name__   ,  unittest.TestCase    ):
	lowercase    =				PegasusTokenizer
	lowercase    =				PegasusTokenizerFast
	lowercase    =				True
	lowercase    =				True





	def 				_lowercase(						self					)    ->    Tuple:
								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCAmelCase  :	int							  =	PegasusTokenizer(A					)
								tokenizer.save_pretrained(self.tmpdirname					)





	@cached_property
	def 				_lowercase(						self					)    ->    int:
								return PegasusTokenizer.from_pretrained("""google/pegasus-large"""					)





	def 				_lowercase(						self	,	**A					)    ->    PegasusTokenizer:
								return PegasusTokenizer.from_pretrained(self.tmpdirname	,	**A					)





	def 				_lowercase(						self	,	A					)    ->    List[str]:
								return ("This is a test", "This is a test")





	def 				_lowercase(						self					)    ->    Optional[Any]:
								UpperCAmelCase  :	Optional[int]							  =	"""</s>"""
								UpperCAmelCase  :	Optional[int]							  =	1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(A					)	,	A					)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(A					)	,	A					)





	def 				_lowercase(						self					)    ->    Tuple:
								UpperCAmelCase  :	str							  =	list(self.get_tokenizer().get_vocab().keys()					)

								self.assertEqual(vocab_keys[0]	,	"""<pad>"""					)
								self.assertEqual(vocab_keys[1]	,	"""</s>"""					)
								self.assertEqual(vocab_keys[-1]	,	"""v"""					)
								self.assertEqual(len(A					)	,	1103					)





	def 				_lowercase(						self					)    ->    Optional[Any]:
								self.assertEqual(self.get_tokenizer().vocab_size	,	1103					)





	def 				_lowercase(						self					)    ->    int:
								UpperCAmelCase  :	Any							  =	self.rust_tokenizer_class.from_pretrained(self.tmpdirname					)
								UpperCAmelCase  :	Optional[Any]							  =	self.tokenizer_class.from_pretrained(self.tmpdirname					)
								UpperCAmelCase  :	Union[str, Any]							  =	(
								    """Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"""
								    """ </s> <pad> <pad> <pad>"""
								)
								UpperCAmelCase  :	Optional[Any]							  =	rust_tokenizer([raw_input_str]	,	return_tensors=A	,	add_special_tokens=A					).input_ids[0]
								UpperCAmelCase  :	Optional[Any]							  =	py_tokenizer([raw_input_str]	,	return_tensors=A	,	add_special_tokens=A					).input_ids[0]
								self.assertListEqual(A	,	A					)





	def 				_lowercase(						self					)    ->    Optional[Any]:
								UpperCAmelCase  :	List[Any]							  =	self._large_tokenizer
								# <mask_1> masks whole sentence while <mask_2> masks single word
								UpperCAmelCase  :	Any							  =	"""<mask_1> To ensure a <mask_2> flow of bank resolutions."""
								UpperCAmelCase  :	Optional[Any]							  =	[2, 413, 615, 114, 3, 1971, 113, 1679, 10710, 107, 1]
								UpperCAmelCase  :	Optional[Any]							  =	tokenizer([raw_input_str]	,	return_tensors=A					).input_ids[0]
								self.assertListEqual(A	,	A					)





	def 				_lowercase(						self					)    ->    int:
								UpperCAmelCase  :	str							  =	self._large_tokenizer
								# The tracebacks for the following asserts are **better** without messages or self.assertEqual
								assert tokenizer.vocab_size == 96103
								assert tokenizer.pad_token_id == 0
								assert tokenizer.eos_token_id == 1
								assert tokenizer.offset == 103
								assert tokenizer.unk_token_id == tokenizer.offset + 2 == 105
								assert tokenizer.unk_token == "<unk>"
								assert tokenizer.model_max_length == 1024
								UpperCAmelCase  :	List[Any]							  =	"""To ensure a smooth flow of bank resolutions."""
								UpperCAmelCase  :	Optional[int]							  =	[413, 615, 114, 2291, 1971, 113, 1679, 10710, 107, 1]
								UpperCAmelCase  :	Union[str, Any]							  =	tokenizer([raw_input_str]	,	return_tensors=A					).input_ids[0]
								self.assertListEqual(A	,	A					)
								assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]					) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]





	@require_torch
	def 				_lowercase(						self					)    ->    Any:
								UpperCAmelCase  :	int							  =	["""This is going to be way too long.""" * 150, """short example"""]
								UpperCAmelCase  :	Optional[int]							  =	["""not super long but more than 5 tokens""", """tiny"""]
								UpperCAmelCase  :	Tuple							  =	self._large_tokenizer(A	,	padding=A	,	truncation=A	,	return_tensors="""pt"""					)
								UpperCAmelCase  :	List[Any]							  =	self._large_tokenizer(
								    text_target=A	,	max_length=5	,	padding=A	,	truncation=A	,	return_tensors="""pt"""					)

								assert batch.input_ids.shape == (2, 1024)
								assert batch.attention_mask.shape == (2, 1024)
								assert targets["input_ids"].shape == (2, 5)
								assert len(A					) == 2  # input_ids, attention_mask.






	@slow
	def 				_lowercase(						self					)    ->    List[str]:
								# fmt: off
								UpperCAmelCase  :	List[str]							  =	{"""input_ids""": [[38979, 143, 18485, 606, 130, 26669, 87686, 121, 54189, 1129, 111, 26669, 87686, 121, 9114, 14787, 121, 13249, 158, 592, 956, 121, 14621, 31576, 143, 62613, 108, 9688, 930, 43430, 11562, 62613, 304, 108, 11443, 897, 108, 9314, 17415, 63399, 108, 11443, 7614, 18316, 118, 4284, 7148, 12430, 143, 1400, 25703, 158, 111, 4284, 7148, 11772, 143, 21297, 1064, 158, 122, 204, 3506, 1754, 1133, 14787, 1581, 115, 33224, 4482, 111, 1355, 110, 29173, 317, 50833, 108, 20147, 94665, 111, 77198, 107, 1], [110, 62613, 117, 638, 112, 1133, 121, 20098, 1355, 79050, 13872, 135, 1596, 53541, 1352, 141, 13039, 5542, 124, 302, 518, 111, 268, 2956, 115, 149, 4427, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [139, 1235, 2799, 18289, 17780, 204, 109, 9474, 1296, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=A	,	model_name="""google/bigbird-pegasus-large-arxiv"""	,	revision="""ba85d0851d708441f91440d509690f1ab6353415"""	,	)




@require_sentencepiece
@require_tokenizers
class     UpperCamelCase_				(							__magic_name__   ,  unittest.TestCase    ):
	lowercase    =				PegasusTokenizer
	lowercase    =				PegasusTokenizerFast
	lowercase    =				True
	lowercase    =				True





	def 				_lowercase(						self					)    ->    List[Any]:
								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCAmelCase  :	int							  =	PegasusTokenizer(A	,	offset=0	,	mask_token_sent=A	,	mask_token="""[MASK]"""					)
								tokenizer.save_pretrained(self.tmpdirname					)





	@cached_property
	def 				_lowercase(						self					)    ->    Optional[Any]:
								return PegasusTokenizer.from_pretrained("""google/bigbird-pegasus-large-arxiv"""					)





	def 				_lowercase(						self	,	**A					)    ->    PegasusTokenizer:
								return PegasusTokenizer.from_pretrained(self.tmpdirname	,	**A					)





	def 				_lowercase(						self	,	A					)    ->    str:
								return ("This is a test", "This is a test")





	def 				_lowercase(						self					)    ->    Union[str, Any]:
								UpperCAmelCase  :	Dict							  =	self.rust_tokenizer_class.from_pretrained(self.tmpdirname					)
								UpperCAmelCase  :	Any							  =	self.tokenizer_class.from_pretrained(self.tmpdirname					)
								UpperCAmelCase  :	str							  =	(
								    """Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"""
								    """ <pad> <pad> <pad>"""
								)
								UpperCAmelCase  :	List[str]							  =	rust_tokenizer([raw_input_str]	,	return_tensors=A	,	add_special_tokens=A					).input_ids[0]
								UpperCAmelCase  :	str							  =	py_tokenizer([raw_input_str]	,	return_tensors=A	,	add_special_tokens=A					).input_ids[0]
								self.assertListEqual(A	,	A					)





	@require_torch
	def 				_lowercase(						self					)    ->    Optional[int]:
								UpperCAmelCase  :	Union[str, Any]							  =	["""This is going to be way too long.""" * 1000, """short example"""]
								UpperCAmelCase  :	Any							  =	["""not super long but more than 5 tokens""", """tiny"""]
								UpperCAmelCase  :	int							  =	self._large_tokenizer(A	,	padding=A	,	truncation=A	,	return_tensors="""pt"""					)
								UpperCAmelCase  :	Optional[int]							  =	self._large_tokenizer(
								    text_target=A	,	max_length=5	,	padding=A	,	truncation=A	,	return_tensors="""pt"""					)

								assert batch.input_ids.shape == (2, 4096)
								assert batch.attention_mask.shape == (2, 4096)
								assert targets["input_ids"].shape == (2, 5)
								assert len(A					) == 2  # input_ids, attention_mask.






	def 				_lowercase(						self					)    ->    int:
								UpperCAmelCase  :	Union[str, Any]							  =	(
								    """This is an example string that is used to test the original TF implementation against the HF"""
								    """ implementation"""
								)

								UpperCAmelCase  :	Optional[Any]							  =	self._large_tokenizer(A					).input_ids

								self.assertListEqual(
								    A	,	[182, 117, 142, 587, 4211, 120, 117, 263, 112, 804, 109, 856, 25016, 3137, 464, 109, 26955, 3137, 1]	,	)

