




import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase	:       Union[str, Any]		    =	get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
class 					snake_case__							(lowercase_					, unittest.TestCase     ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_ :							Optional[int]											=      XLMProphetNetTokenizer
				SCREAMING_SNAKE_CASE_ :							Optional[Any]											=      False
				SCREAMING_SNAKE_CASE_ :							Optional[Any]											=      True





				def 							__UpperCAmelCase							(   self       :  Tuple							)      ->  List[Any]:
							super().setUp()

							# We have a SentencePiece fixture for testing
							a					=    XLMProphetNetTokenizer(UpperCamelCase__      ,   keep_accents=UpperCamelCase__							)
							tokenizer.save_pretrained(self.tmpdirname							)





				def 							__UpperCAmelCase							(   self       :  Dict							)      ->  str:
							a					=    "[PAD]"
							a					=    0

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__							)      ,   UpperCamelCase__							)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__							)      ,   UpperCamelCase__							)





				def 							__UpperCAmelCase							(   self       :  Any							)      ->  Dict:
							a					=    list(self.get_tokenizer().get_vocab().keys()							)

							self.assertEqual(vocab_keys[0]      ,   "[PAD]"							)
							self.assertEqual(vocab_keys[1]      ,   "[CLS]"							)
							self.assertEqual(vocab_keys[-1]      ,   "j"							)
							self.assertEqual(len(UpperCamelCase__							)      ,   10_12							)





				def 							__UpperCAmelCase							(   self       :  Optional[Any]							)      ->  int:
							self.assertEqual(self.get_tokenizer().vocab_size      ,   10_12							)





				def 							__UpperCAmelCase							(   self       :  Any							)      ->  str:
							a					=    XLMProphetNetTokenizer(UpperCamelCase__      ,   keep_accents=UpperCamelCase__							)

							a					=    tokenizer.tokenize("This is a test"							)
							self.assertListEqual(UpperCamelCase__      ,   ["▁This", "▁is", "▁a", "▁t", "est"]							)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(UpperCamelCase__							)      ,   [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]      ,   )

							a					=    tokenizer.tokenize("I was born in 92000, and this is falsé."							)
							self.assertListEqual(
							    UpperCamelCase__      ,   [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "é",
							        ".",
							    ]      ,   )
							a					=    tokenizer.convert_tokens_to_ids(UpperCamelCase__							)
							self.assertListEqual(
							    UpperCamelCase__      ,   [
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, -9, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, -9, 4]
							    ]      ,   )

							a					=    tokenizer.convert_ids_to_tokens(UpperCamelCase__							)
							self.assertListEqual(
							    UpperCamelCase__      ,   [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "[UNK]",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "[UNK]",
							        ".",
							    ]      ,   )





				@cached_property
				def 							__UpperCAmelCase							(   self       :  Union[str, Any]							)      ->  Tuple:
							return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"							)





				@slow
				def 							__UpperCAmelCase							(   self       :  Dict							)      ->  List[str]:
							a					=    "Hello World!"
							a					=    [3_53_89, 66_72, 49, 2]
							self.assertListEqual(UpperCamelCase__      ,   self.big_tokenizer.encode(UpperCamelCase__							)							)





				@slow
				def 							__UpperCAmelCase							(   self       :  Dict							)      ->  List[str]:
							a					=    {"input_ids": [[1_10_73, 8_27_83, 18, 26, 8_27_83, 5_49, 5_15_40, 2_48, 1_72_09, 13_01, 2_17, 20, 21_51_86, 13_25, 1_47, 1_72_09, 13_01, 2_17, 20, 5_63_70, 53, 12_20_20, 20, 1_64_77, 27, 8_73_55, 45_48, 20, 47_28, 7_83_92, 17, 15_99_69, 18, 26, 2_44_91, 6_29, 15, 5_38, 2_27_04, 54_39, 15, 27_88, 2_44_91, 98_85, 15, 4_35_34, 6_05, 15, 8_14, 1_84_03, 3_32_00, 29, 15, 4_35_34, 2_44_58, 1_24_10, 1_11, 2_49_66, 8_36_69, 96_37, 14_40_68, 26, 8_50, 2_23_46, 27, 1_47, 2_49_66, 8_36_69, 8_34_90, 26, 3_91_13, 7_35, 27, 6_89, 6_56, 28_00, 13_39, 46_00, 53, 12_20_20, 11_57_85, 34, 8_16, 13_39, 4_68_87, 18, 1_47, 5_39_05, 19_51, 4_22_38, 4_11_70, 1_77_32, 8_34, 4_36, 15, 2_75_23, 9_87_33, 2_17, 1_47, 55_42, 49_81, 9_30, 1_73_47, 16, 2], [2_00_91, 6_29, 94, 8_27_86, 58, 4_90, 20, 15_28, 84, 5_39_05, 3_44, 8_05_92, 11_01_28, 1_88_22, 52_67, 13_06, 62, 15_25_37, 3_08, 79_97, 4_01, 12_44_27, 5_49, 3_54_42, 2_25, 1_09, 1_50_55, 2_57_48, 1_47, 71_19, 4_37_12, 34, 7_67, 13_53_66, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_92, 6_37_84, 11_94_66, 17, 14_78_08, 8_82_14, 18, 6_56, 81, 32, 32_96, 1_02_80, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=UpperCamelCase__      ,   model_name="microsoft/xprophetnet-large-wiki100-cased"      ,   revision="1acad1643ddd54a44df6a1b797ada8373685d90e"      ,   )












import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_big_bird import BigBirdTokenizer
else:
		__lowerCAmelCase	:       Optional[int]     =	None

__lowerCAmelCase	:       Optional[Any]     =	logging.get_logger(__name__)
__lowerCAmelCase	:       Optional[int]     =	{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase	:       List[Any]     =	{
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    },
    'tokenizer_file': {
        'google/bigbird-roberta-base': (
            'https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json'
        ),
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json'
        ),
    },
}

__lowerCAmelCase	:       List[str]     =	{
    'google/bigbird-roberta-base': 4096,
    'google/bigbird-roberta-large': 4096,
    'google/bigbird-base-trivia-itc': 4096,
}


__lowerCAmelCase	:       Any     =	'▁'




class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_ :							Any											=      VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_ :							List[Any]											=      PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_ :							Dict											=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_ :							str											=      BigBirdTokenizer
				SCREAMING_SNAKE_CASE_ :							str											=      ["""input_ids""", """attention_mask"""]
				SCREAMING_SNAKE_CASE_ :							List[int]											=      []





				def __init__(   self       :  int      ,   __lowerCamelCase       :  Any=None      ,   __lowerCamelCase       :  List[str]=None      ,   __lowerCamelCase       :  Optional[int]="<unk>"      ,   __lowerCamelCase       :  int="<s>"      ,   __lowerCamelCase       :  Optional[Any]="</s>"      ,   __lowerCamelCase       :  Tuple="<pad>"      ,   __lowerCamelCase       :  Tuple="[SEP]"      ,   __lowerCamelCase       :  Dict="[MASK]"      ,   __lowerCamelCase       :  Tuple="[CLS]"      ,   **__lowerCamelCase       :  Optional[Any]      ,   )      ->  List[Any]:
							a					=    AddedToken(__lowerCamelCase      ,   lstrip=__lowerCamelCase      ,   rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase      ,   __lowerCamelCase							) else bos_token
							a					=    AddedToken(__lowerCamelCase      ,   lstrip=__lowerCamelCase      ,   rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase      ,   __lowerCamelCase							) else eos_token
							a					=    AddedToken(__lowerCamelCase      ,   lstrip=__lowerCamelCase      ,   rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase      ,   __lowerCamelCase							) else unk_token
							a					=    AddedToken(__lowerCamelCase      ,   lstrip=__lowerCamelCase      ,   rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase      ,   __lowerCamelCase							) else pad_token
							a					=    AddedToken(__lowerCamelCase      ,   lstrip=__lowerCamelCase      ,   rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase      ,   __lowerCamelCase							) else cls_token
							a					=    AddedToken(__lowerCamelCase      ,   lstrip=__lowerCamelCase      ,   rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase      ,   __lowerCamelCase							) else sep_token

							# Mask token behave like a normal word, i.e. include the space before it
							a					=    AddedToken(__lowerCamelCase      ,   lstrip=__lowerCamelCase      ,   rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase      ,   __lowerCamelCase							) else mask_token

							super().__init__(
							    __lowerCamelCase      ,   tokenizer_file=__lowerCamelCase      ,   bos_token=__lowerCamelCase      ,   eos_token=__lowerCamelCase      ,   unk_token=__lowerCamelCase      ,   sep_token=__lowerCamelCase      ,   pad_token=__lowerCamelCase      ,   cls_token=__lowerCamelCase      ,   mask_token=__lowerCamelCase      ,   **__lowerCamelCase      ,   )

							a					=    vocab_file
							a					=    False if not self.vocab_file else True





				def 							__UpperCAmelCase							(   self       :  str      ,   __lowerCamelCase       :  List[int]      ,   __lowerCamelCase       :  Optional[List[int]] = None							)      ->  List[int]:
							a					=    [self.sep_token_id]
							a					=    [self.cls_token_id]
							if token_ids_a is None:
										return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep





				def 							__UpperCAmelCase							(   self       :  List[str]      ,   __lowerCamelCase       :  List[int]      ,   __lowerCamelCase       :  Optional[List[int]] = None      ,   __lowerCamelCase       :  bool = False							)      ->  List[int]:

							if already_has_special_tokens:
										if token_ids_a is not None:
													raise ValueError(
													    "You should not supply a second sequence if the provided sequence of "
													    "ids is already formatted with special tokens for the model."							)
										return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

							if token_ids_a is None:
										return [1] + ([0] * len(__lowerCamelCase							)) + [1]
							return [1] + ([0] * len(__lowerCamelCase							)) + [1] + ([0] * len(__lowerCamelCase							)) + [1]





				def 							__UpperCAmelCase							(   self       :  Optional[int]      ,   __lowerCamelCase       :  List[int]      ,   __lowerCamelCase       :  Optional[List[int]] = None							)      ->  List[int]:
							a					=    [self.sep_token_id]
							a					=    [self.cls_token_id]

							if token_ids_a is None:
										return len(cls + token_ids_a + sep							) * [0]
							return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]





				def 							__UpperCAmelCase							(   self       :  Tuple      ,   __lowerCamelCase       :  str      ,   __lowerCamelCase       :  Optional[str] = None							)      ->  Tuple[str]:
							if not self.can_save_slow_tokenizer:
										raise ValueError(
										    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
										    "tokenizer."							)

							if not os.path.isdir(__lowerCamelCase							):
										logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
										return
							a					=    os.path.join(
							    __lowerCamelCase      ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]							)

							if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowerCamelCase							):
										copyfile(self.vocab_file      ,   __lowerCamelCase							)

							return (out_vocab_file,)






