


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


__a     =			logging.get_logger(__name__)

__a     =			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__a     =			{
    """vocab_file""": {
        """yjernite/retribert-base-uncased""": (
            """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """yjernite/retribert-base-uncased""": (
            """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

__a     =			{
    """yjernite/retribert-base-uncased""": 512,
}


__a     =			{
    """yjernite/retribert-base-uncased""": {"""do_lower_case""": True},
}






class   __lowercase				(  __snake_case  ):
      UpperCamelCase								=       VOCAB_FILES_NAMES
      UpperCamelCase								=       PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase								=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase								=       PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase								=       RetriBertTokenizer
      UpperCamelCase								=       ['''input_ids''', '''attention_mask''']





      def __init__(		self						:					Optional[int]					,       __lowerCamelCase						:					Tuple=None					,       __lowerCamelCase						:					str=None					,       __lowerCamelCase						:					Optional[Any]=True					,       __lowerCamelCase						:					int="[UNK]"					,       __lowerCamelCase						:					Dict="[SEP]"					,       __lowerCamelCase						:					Union[str, Any]="[PAD]"					,       __lowerCamelCase						:					Optional[Any]="[CLS]"					,       __lowerCamelCase						:					str="[MASK]"					,       __lowerCamelCase						:					Union[str, Any]=True					,       __lowerCamelCase						:					int=None					,       **__lowerCamelCase						:					Any					,       )    ->   Any:






         """simple docstring"""







         super().__init__(
             __lowerCamelCase					,       tokenizer_file=__lowerCamelCase					,       do_lower_case=__lowerCamelCase					,       unk_token=__lowerCamelCase					,       sep_token=__lowerCamelCase					,       pad_token=__lowerCamelCase					,       cls_token=__lowerCamelCase					,       mask_token=__lowerCamelCase					,       tokenize_chinese_chars=__lowerCamelCase					,       strip_accents=__lowerCamelCase					,       **__lowerCamelCase					,       )

         UpperCAmelCase							=     json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
         if (
             normalizer_state.get("""lowercase"""					,       __lowerCamelCase       ) != do_lower_case
             or normalizer_state.get("""strip_accents"""					,       __lowerCamelCase       ) != strip_accents
             or normalizer_state.get("""handle_chinese_chars"""					,       __lowerCamelCase       ) != tokenize_chinese_chars
         ):
            UpperCAmelCase							=     getattr(__lowerCamelCase					,       normalizer_state.pop("""type"""       )       )
            UpperCAmelCase							=     do_lower_case
            UpperCAmelCase							=     strip_accents
            UpperCAmelCase							=     tokenize_chinese_chars
            UpperCAmelCase							=     normalizer_class(**__lowerCamelCase       )

         UpperCAmelCase							=     do_lower_case





      def        _lowercase				(		self						:					int					,       __lowerCamelCase						:					str					,       __lowerCamelCase						:					Dict=None       )    ->   Optional[int]:






         """simple docstring"""







         UpperCAmelCase							=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

         return output





      def        _lowercase				(		self						:					Optional[Any]					,       __lowerCamelCase						:					List[int]					,       __lowerCamelCase						:					Optional[List[int]] = None       )    ->   List[int]:






         """simple docstring"""







         UpperCAmelCase							=     [self.sep_token_id]
         UpperCAmelCase							=     [self.cls_token_id]
         if token_ids_a is None:
            return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





      def        _lowercase				(		self						:					Optional[int]					,       __lowerCamelCase						:					str					,       __lowerCamelCase						:					Optional[str] = None       )    ->   Tuple[str]:






         """simple docstring"""







         UpperCAmelCase							=     self._tokenizer.model.save(__lowerCamelCase					,       name=__lowerCamelCase       )
         return tuple(__lowerCamelCase       )








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__a     =			{"""configuration_ibert""": ["""IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """IBertConfig""", """IBertOnnxConfig"""]}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __a     =			[
     """IBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """IBertForMaskedLM""",
     """IBertForMultipleChoice""",
     """IBertForQuestionAnswering""",
     """IBertForSequenceClassification""",
     """IBertForTokenClassification""",
     """IBertModel""",
     """IBertPreTrainedModel""",
 ]

if TYPE_CHECKING:
 from .configuration_ibert import IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, IBertConfig, IBertOnnxConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_ibert import (
      IBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      IBertForMaskedLM,
      IBertForMultipleChoice,
      IBertForQuestionAnswering,
      IBertForSequenceClassification,
      IBertForTokenClassification,
      IBertModel,
      IBertPreTrainedModel,
  )

else:
 import sys

 __a     =			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



