





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_big_bird import BigBirdTokenizer
else:
  SCREAMING_SNAKE_CASE__       :			List[str]			      =						None

SCREAMING_SNAKE_CASE__       :			Dict			      =						logging.get_logger(__name__)
SCREAMING_SNAKE_CASE__       :			str			      =						{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE__       :			List[str]			      =						{
    """vocab_file""": {
        """google/bigbird-roberta-base""": """https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model""",
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"""
        ),
    },
    """tokenizer_file""": {
        """google/bigbird-roberta-base""": (
            """https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json"""
        ),
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE__       :			Union[str, Any]			      =						{
    """google/bigbird-roberta-base""": 4_0_9_6,
    """google/bigbird-roberta-large""": 4_0_9_6,
    """google/bigbird-base-trivia-itc""": 4_0_9_6,
}


SCREAMING_SNAKE_CASE__       :			str			      =						"""▁"""







class   __lowerCAmelCase			(					__A  ):
   _UpperCamelCase :						Union[str, Any]     		=				VOCAB_FILES_NAMES
   _UpperCamelCase :						List[Any]     		=				PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase :						Union[str, Any]     		=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase :						Dict     		=				BigBirdTokenizer
   _UpperCamelCase :						Tuple     		=				["""input_ids""", """attention_mask"""]
   _UpperCamelCase :						Optional[int]     		=				[]





   def __init__( self					,			snake_case=None					,			snake_case=None					,			snake_case="<unk>"					,			snake_case="<s>"					,			snake_case="</s>"					,			snake_case="<pad>"					,			snake_case="[SEP]"					,			snake_case="[MASK]"					,			snake_case="[CLS]"					,			**snake_case					,			)				->  List[str]:
      """simple docstring"""





      a__	:						List[Any]      		=  AddedToken(snake_case					,			lstrip=snake_case					,			rstrip=snake_case      ) if isinstance(snake_case					,			snake_case      ) else bos_token
      a__	:						Optional[Any]      		=  AddedToken(snake_case					,			lstrip=snake_case					,			rstrip=snake_case      ) if isinstance(snake_case					,			snake_case      ) else eos_token
      a__	:						List[str]      		=  AddedToken(snake_case					,			lstrip=snake_case					,			rstrip=snake_case      ) if isinstance(snake_case					,			snake_case      ) else unk_token
      a__	:						List[str]      		=  AddedToken(snake_case					,			lstrip=snake_case					,			rstrip=snake_case      ) if isinstance(snake_case					,			snake_case      ) else pad_token
      a__	:						List[Any]      		=  AddedToken(snake_case					,			lstrip=snake_case					,			rstrip=snake_case      ) if isinstance(snake_case					,			snake_case      ) else cls_token
      a__	:						Dict      		=  AddedToken(snake_case					,			lstrip=snake_case					,			rstrip=snake_case      ) if isinstance(snake_case					,			snake_case      ) else sep_token

      # Mask token behave like a normal word, i.e. include the space before it
      a__	:						Optional[int]      		=  AddedToken(snake_case					,			lstrip=snake_case					,			rstrip=snake_case      ) if isinstance(snake_case					,			snake_case      ) else mask_token

      super().__init__(
          snake_case					,			tokenizer_file=snake_case					,			bos_token=snake_case					,			eos_token=snake_case					,			unk_token=snake_case					,			sep_token=snake_case					,			pad_token=snake_case					,			cls_token=snake_case					,			mask_token=snake_case					,			**snake_case					,			)

      a__	:						List[str]      		=  vocab_file
      a__	:						Dict      		=  False if not self.vocab_file else True





   def 	_snake_case ( self					,			snake_case					,			snake_case = None      )				->  List[Any]:
      """simple docstring"""





      a__	:						List[Any]      		=  [self.sep_token_id]
      a__	:						Dict      		=  [self.cls_token_id]
      if token_ids_a is None:
         return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep





   def 	_snake_case ( self					,			snake_case					,			snake_case = None					,			snake_case = False      )				->  int:
      """simple docstring"""





      if already_has_special_tokens:
         if token_ids_a is not None:
            raise ValueError(
                "You should not supply a second sequence if the provided sequence of "
                "ids is already formatted with special tokens for the model."      )
         return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

      if token_ids_a is None:
         return [1] + ([0] * len(snake_case      )) + [1]
      return [1] + ([0] * len(snake_case      )) + [1] + ([0] * len(snake_case      )) + [1]





   def 	_snake_case ( self					,			snake_case					,			snake_case = None      )				->  Union[str, Any]:
      """simple docstring"""





      a__	:						Optional[Any]      		=  [self.sep_token_id]
      a__	:						Dict      		=  [self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep      ) * [0]
      return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]





   def 	_snake_case ( self					,			snake_case					,			snake_case = None      )				->  Optional[Any]:
      """simple docstring"""





      if not self.can_save_slow_tokenizer:
         raise ValueError(
             "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
             "tokenizer."      )

      if not os.path.isdir(snake_case      ):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
         return
      a__	:						Optional[Any]      		=  os.path.join(
          snake_case					,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )

      if os.path.abspath(self.vocab_file      ) != os.path.abspath(snake_case      ):
         copyfile(self.vocab_file					,			snake_case      )

      return (out_vocab_file,)








import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


SCREAMING_SNAKE_CASE__       :			Dict			      =						"""
Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.

In March 2021, Hugging Face raised $40 million in a Series B funding round.[3]

On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]
"""







class   __lowerCAmelCase			(					unittest.TestCase		,_UpperCamelCase  ):





   def 	_snake_case ( self      )				->  str:
      """simple docstring"""





      a__	:						Optional[int]      		=  load_tool("text-question-answering"      )
      self.tool.setup()
      a__	:						Dict      		=  load_tool("text-question-answering"					,			remote=snake_case      )





   def 	_snake_case ( self      )				->  Dict:
      """simple docstring"""





      a__	:						Optional[Any]      		=  self.tool(snake_case					,			"What did Hugging Face do in April 2021?"      )
      self.assertEqual(snake_case					,			"launched the BigScience Research Workshop"      )





   def 	_snake_case ( self      )				->  Tuple:
      """simple docstring"""





      a__	:						List[Any]      		=  self.remote_tool(snake_case					,			"What did Hugging Face do in April 2021?"      )
      self.assertEqual(snake_case					,			"launched the BigScience Research Workshop"      )





   def 	_snake_case ( self      )				->  Any:
      """simple docstring"""





      a__	:						Any      		=  self.tool(text=snake_case					,			question="What did Hugging Face do in April 2021?"      )
      self.assertEqual(snake_case					,			"launched the BigScience Research Workshop"      )





   def 	_snake_case ( self      )				->  int:
      """simple docstring"""





      a__	:						List[str]      		=  self.remote_tool(text=snake_case					,			question="What did Hugging Face do in April 2021?"      )
      self.assertEqual(snake_case					,			"launched the BigScience Research Workshop"      )

