


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_big_bird import BigBirdTokenizer
else:
							a					  =     None

a					  =     logging.get_logger(__name__)
a					  =     {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

a					  =     {
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    },
    'tokenizer_file': {
        'google/bigbird-roberta-base': (
            'https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json'
        ),
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json'
        ),
    },
}

a					  =     {
    'google/bigbird-roberta-base': 4_0_9_6,
    'google/bigbird-roberta-large': 4_0_9_6,
    'google/bigbird-base-trivia-itc': 4_0_9_6,
}


a					  =     '▁'







class        UpperCamelCase__   (	lowerCamelCase__   ):
					__SCREAMING_SNAKE_CASE       :  Optional[int]						=   VOCAB_FILES_NAMES
					__SCREAMING_SNAKE_CASE       :  Dict						=   PRETRAINED_VOCAB_FILES_MAP
					__SCREAMING_SNAKE_CASE       :  List[str]						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__SCREAMING_SNAKE_CASE       :  str						=   BigBirdTokenizer
					__SCREAMING_SNAKE_CASE       :  Any						=   ['input_ids', 'attention_mask']
					__SCREAMING_SNAKE_CASE       :  Optional[Any]						=   []

					def __init__(     self       :   List[Any]							,					UpperCamelCase__       :   Optional[Any]=None							,					UpperCamelCase__       :   Tuple=None							,					UpperCamelCase__       :   Any="<unk>"							,					UpperCamelCase__       :   Optional[Any]="<s>"							,					UpperCamelCase__       :   str="</s>"							,					UpperCamelCase__       :   Optional[Any]="<pad>"							,					UpperCamelCase__       :   Union[str, Any]="[SEP]"							,					UpperCamelCase__       :   Tuple="[MASK]"							,					UpperCamelCase__       :   List[str]="[CLS]"							,					**UpperCamelCase__       :   Union[str, Any]							,					):






										'''simple docstring'''






										lowercase_  							=			AddedToken(__lowerCamelCase							,					lstrip=__lowerCamelCase							,					rstrip=__lowerCamelCase       ) if isinstance(__lowerCamelCase							,					__lowerCamelCase       ) else bos_token
										lowercase_  							=			AddedToken(__lowerCamelCase							,					lstrip=__lowerCamelCase							,					rstrip=__lowerCamelCase       ) if isinstance(__lowerCamelCase							,					__lowerCamelCase       ) else eos_token
										lowercase_  							=			AddedToken(__lowerCamelCase							,					lstrip=__lowerCamelCase							,					rstrip=__lowerCamelCase       ) if isinstance(__lowerCamelCase							,					__lowerCamelCase       ) else unk_token
										lowercase_  							=			AddedToken(__lowerCamelCase							,					lstrip=__lowerCamelCase							,					rstrip=__lowerCamelCase       ) if isinstance(__lowerCamelCase							,					__lowerCamelCase       ) else pad_token
										lowercase_  							=			AddedToken(__lowerCamelCase							,					lstrip=__lowerCamelCase							,					rstrip=__lowerCamelCase       ) if isinstance(__lowerCamelCase							,					__lowerCamelCase       ) else cls_token
										lowercase_  							=			AddedToken(__lowerCamelCase							,					lstrip=__lowerCamelCase							,					rstrip=__lowerCamelCase       ) if isinstance(__lowerCamelCase							,					__lowerCamelCase       ) else sep_token

										# Mask token behave like a normal word, i.e. include the space before it
										lowercase_  							=			AddedToken(__lowerCamelCase							,					lstrip=__lowerCamelCase							,					rstrip=__lowerCamelCase       ) if isinstance(__lowerCamelCase							,					__lowerCamelCase       ) else mask_token

										super().__init__(
										    __lowerCamelCase							,					tokenizer_file=__lowerCamelCase							,					bos_token=__lowerCamelCase							,					eos_token=__lowerCamelCase							,					unk_token=__lowerCamelCase							,					sep_token=__lowerCamelCase							,					pad_token=__lowerCamelCase							,					cls_token=__lowerCamelCase							,					mask_token=__lowerCamelCase							,					**__lowerCamelCase							,					)

										lowercase_  							=			vocab_file
										lowercase_  							=			False if not self.vocab_file else True

					def   UpperCAmelCase__       (     self       :   Dict							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   Optional[Any] = None       ):






										'''simple docstring'''






										lowercase_  							=			[self.sep_token_id]
										lowercase_  							=			[self.cls_token_id]
										if token_ids_a is None:
															return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep

					def   UpperCAmelCase__       (     self       :   Any							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   Union[str, Any] = None							,					UpperCamelCase__       :   Optional[Any] = False       ):






										'''simple docstring'''






										if already_has_special_tokens:
															if token_ids_a is not None:
																				raise ValueError(
																				    """You should not supply a second sequence if the provided sequence of """
																				    """ids is already formatted with special tokens for the model."""       )
															return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

										if token_ids_a is None:
															return [1] + ([0] * len(__lowerCamelCase       )) + [1]
										return [1] + ([0] * len(__lowerCamelCase       )) + [1] + ([0] * len(__lowerCamelCase       )) + [1]

					def   UpperCAmelCase__       (     self       :   Tuple							,					UpperCamelCase__       :   List[Any]							,					UpperCamelCase__       :   str = None       ):






										'''simple docstring'''






										lowercase_  							=			[self.sep_token_id]
										lowercase_  							=			[self.cls_token_id]

										if token_ids_a is None:
															return len(cls + token_ids_a + sep       ) * [0]
										return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







					def   UpperCAmelCase__       (     self       :   Dict							,					UpperCamelCase__       :   List[str]							,					UpperCamelCase__       :   Tuple = None       ):






										'''simple docstring'''






										if not self.can_save_slow_tokenizer:
															raise ValueError(
															    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
															    """tokenizer."""       )

										if not os.path.isdir(__lowerCamelCase       ):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
															return
										lowercase_  							=			os.path.join(
										    __lowerCamelCase							,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )

										if os.path.abspath(self.vocab_file       ) != os.path.abspath(__lowerCamelCase       ):
															copyfile(self.vocab_file							,					__lowerCamelCase       )

										return (out_vocab_file,)



def 			UpperCAmelCase_				(      UpperCAmelCase__  ):
					if not isinstance(UpperCAmelCase__						,							UpperCAmelCase__  ):
										raise TypeError("""Input value must be an 'int' type"""  )

					lowercase_  							=			0
					while number:
										position += 1
										number >>= 1

					return position


if __name__ == "__main__":
							import doctest

							doctest.testmod()


