





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_big_bird import BigBirdTokenizer
else:
  SCREAMING_SNAKE_CASE__       :			Any			      =						None

SCREAMING_SNAKE_CASE__       :			List[Any]			      =						logging.get_logger(__name__)
SCREAMING_SNAKE_CASE__       :			List[Any]			      =						{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__       :			Optional[int]			      =						{
    """vocab_file""": {
        """google/bigbird-roberta-base""": """https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model""",
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"""
        ),
    },
    """tokenizer_file""": {
        """google/bigbird-roberta-base""": (
            """https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json"""
        ),
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE__       :			Optional[Any]			      =						{
    """google/bigbird-roberta-base""": 4_0_9_6,
    """google/bigbird-roberta-large""": 4_0_9_6,
    """google/bigbird-base-trivia-itc""": 4_0_9_6,
}


SCREAMING_SNAKE_CASE__       :			int			      =						"""▁"""







class   __lowerCAmelCase			(					UpperCAmelCase__  ):
   _UpperCamelCase :						Any     		=				VOCAB_FILES_NAMES
   _UpperCamelCase :						Optional[Any]     		=				PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase :						Any     		=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase :						List[str]     		=				BigBirdTokenizer
   _UpperCamelCase :						str     		=				["input_ids", "attention_mask"]
   _UpperCamelCase :						List[int]     		=				[]





   def __init__( self					,			snake_case=None					,			snake_case=None					,			snake_case="<unk>"					,			snake_case="<s>"					,			snake_case="</s>"					,			snake_case="<pad>"					,			snake_case="[SEP]"					,			snake_case="[MASK]"					,			snake_case="[CLS]"					,			**snake_case					,			)				->  Dict:
      """simple docstring"""





      a__	:						Dict      		=  AddedToken(lowerCamelCase__					,			lstrip=lowerCamelCase__					,			rstrip=lowerCamelCase__      ) if isinstance(lowerCamelCase__					,			lowerCamelCase__      ) else bos_token
      a__	:						Optional[Any]      		=  AddedToken(lowerCamelCase__					,			lstrip=lowerCamelCase__					,			rstrip=lowerCamelCase__      ) if isinstance(lowerCamelCase__					,			lowerCamelCase__      ) else eos_token
      a__	:						Any      		=  AddedToken(lowerCamelCase__					,			lstrip=lowerCamelCase__					,			rstrip=lowerCamelCase__      ) if isinstance(lowerCamelCase__					,			lowerCamelCase__      ) else unk_token
      a__	:						List[str]      		=  AddedToken(lowerCamelCase__					,			lstrip=lowerCamelCase__					,			rstrip=lowerCamelCase__      ) if isinstance(lowerCamelCase__					,			lowerCamelCase__      ) else pad_token
      a__	:						Tuple      		=  AddedToken(lowerCamelCase__					,			lstrip=lowerCamelCase__					,			rstrip=lowerCamelCase__      ) if isinstance(lowerCamelCase__					,			lowerCamelCase__      ) else cls_token
      a__	:						List[str]      		=  AddedToken(lowerCamelCase__					,			lstrip=lowerCamelCase__					,			rstrip=lowerCamelCase__      ) if isinstance(lowerCamelCase__					,			lowerCamelCase__      ) else sep_token

      # Mask token behave like a normal word, i.e. include the space before it
      a__	:						Any      		=  AddedToken(lowerCamelCase__					,			lstrip=lowerCamelCase__					,			rstrip=lowerCamelCase__      ) if isinstance(lowerCamelCase__					,			lowerCamelCase__      ) else mask_token

      super().__init__(
          lowerCamelCase__					,			tokenizer_file=lowerCamelCase__					,			bos_token=lowerCamelCase__					,			eos_token=lowerCamelCase__					,			unk_token=lowerCamelCase__					,			sep_token=lowerCamelCase__					,			pad_token=lowerCamelCase__					,			cls_token=lowerCamelCase__					,			mask_token=lowerCamelCase__					,			**lowerCamelCase__					,			)

      a__	:						Optional[int]      		=  vocab_file
      a__	:						Optional[Any]      		=  False if not self.vocab_file else True





   def 	_snake_case ( self					,			snake_case					,			snake_case = None      )				->  List[int]:
      """simple docstring"""





      a__	:						List[Any]      		=  [self.sep_token_id]
      a__	:						Optional[Any]      		=  [self.cls_token_id]
      if token_ids_a is None:
         return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep





   def 	_snake_case ( self					,			snake_case					,			snake_case = None					,			snake_case = False      )				->  List[int]:
      """simple docstring"""





      if already_has_special_tokens:
         if token_ids_a is not None:
            raise ValueError(
                "You should not supply a second sequence if the provided sequence of "
                "ids is already formatted with special tokens for the model."      )
         return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

      if token_ids_a is None:
         return [1] + ([0] * len(lowerCamelCase__      )) + [1]
      return [1] + ([0] * len(lowerCamelCase__      )) + [1] + ([0] * len(lowerCamelCase__      )) + [1]





   def 	_snake_case ( self					,			snake_case					,			snake_case = None      )				->  List[int]:
      """simple docstring"""





      a__	:						List[Any]      		=  [self.sep_token_id]
      a__	:						List[str]      		=  [self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep      ) * [0]
      return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]





   def 	_snake_case ( self					,			snake_case					,			snake_case = None      )				->  Tuple[str]:
      """simple docstring"""





      if not self.can_save_slow_tokenizer:
         raise ValueError(
             "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
             "tokenizer."      )

      if not os.path.isdir(lowerCamelCase__      ):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
         return
      a__	:						str      		=  os.path.join(
          lowerCamelCase__					,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )

      if os.path.abspath(self.vocab_file      ) != os.path.abspath(lowerCamelCase__      ):
         copyfile(self.vocab_file					,			lowerCamelCase__      )

      return (out_vocab_file,)










import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


SCREAMING_SNAKE_CASE__       :			Optional[Any]			      =						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__       :			List[Any]			      =						{
    """vocab_file""": {
        """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """unc-nlp/lxmert-base-uncased""": (
            """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE__       :			Any			      =						{
    """unc-nlp/lxmert-base-uncased""": 5_1_2,
}

SCREAMING_SNAKE_CASE__       :			Optional[int]			      =						{
    """unc-nlp/lxmert-base-uncased""": {"""do_lower_case""": True},
}







class   __lowerCAmelCase			(					_UpperCamelCase  ):
   _UpperCamelCase :						List[str]     		=				VOCAB_FILES_NAMES
   _UpperCamelCase :						str     		=				PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase :						List[str]     		=				PRETRAINED_INIT_CONFIGURATION
   _UpperCamelCase :						Optional[int]     		=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase :						Optional[Any]     		=				LxmertTokenizer





   def __init__( self					,			snake_case=None					,			snake_case=None					,			snake_case=True					,			snake_case="[UNK]"					,			snake_case="[SEP]"					,			snake_case="[PAD]"					,			snake_case="[CLS]"					,			snake_case="[MASK]"					,			snake_case=True					,			snake_case=None					,			**snake_case					,			)				->  Any:
      """simple docstring"""





      super().__init__(
          snake_case					,			tokenizer_file=snake_case					,			do_lower_case=snake_case					,			unk_token=snake_case					,			sep_token=snake_case					,			pad_token=snake_case					,			cls_token=snake_case					,			mask_token=snake_case					,			tokenize_chinese_chars=snake_case					,			strip_accents=snake_case					,			**snake_case					,			)

      a__	:						Optional[Any]      		=  json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
      if (
          normalizer_state.get("lowercase"					,			snake_case      ) != do_lower_case
          or normalizer_state.get("strip_accents"					,			snake_case      ) != strip_accents
          or normalizer_state.get("handle_chinese_chars"					,			snake_case      ) != tokenize_chinese_chars
      ):
         a__	:						str      		=  getattr(snake_case					,			normalizer_state.pop("type"      )      )
         a__	:						Tuple      		=  do_lower_case
         a__	:						Union[str, Any]      		=  strip_accents
         a__	:						str      		=  tokenize_chinese_chars
         a__	:						List[str]      		=  normalizer_class(**snake_case      )

      a__	:						str      		=  do_lower_case





   def 	_snake_case ( self					,			snake_case					,			snake_case=None      )				->  List[str]:
      """simple docstring"""





      a__	:						Optional[int]      		=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

      return output





   def 	_snake_case ( self					,			snake_case					,			snake_case = None      )				->  List[int]:
      """simple docstring"""





      a__	:						Tuple      		=  [self.sep_token_id]
      a__	:						str      		=  [self.cls_token_id]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep      ) * [0]
      return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]





   def 	_snake_case ( self					,			snake_case					,			snake_case = None      )				->  Tuple[str]:
      """simple docstring"""





      a__	:						Optional[int]      		=  self._tokenizer.model.save(snake_case					,			name=snake_case      )
      return tuple(snake_case      )

