

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCamelCase__  :      Tuple			=    logging.get_logger(__name__)

UpperCamelCase__  :      int			=    {"vocab_file": "spiece.model"}

UpperCamelCase__  :      str			=    {
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

UpperCamelCase__  :      Union[str, Any]			=    {
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
UpperCamelCase__  :      List[Any]			=    0
UpperCamelCase__  :      Optional[Any]			=    1
UpperCamelCase__  :      Dict			=    2
UpperCamelCase__  :      Tuple			=    3
UpperCamelCase__  :      Optional[int]			=    4



class       __snake_case  (    lowerCAmelCase__       ):
  __lowerCAmelCase     :	Tuple						 =  VOCAB_FILES_NAMES
  __lowerCAmelCase     :	Union[str, Any]						 =  PRETRAINED_VOCAB_FILES_MAP
  __lowerCAmelCase     :	Union[str, Any]						 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowerCAmelCase     :	Optional[int]						 =  'left'

  def __init__(					self						,		_A						,		_A=False						,		_A=True						,		_A=False						,		_A="<s>"						,		_A="</s>"						,		_A="<unk>"						,		_A="<sep>"						,		_A="<pad>"						,		_A="<cls>"						,		_A="<mask>"						,		_A=["<eop>", "<eod>"]						,		_A = None						,		**_A						,		):
      # Mask token behave like a normal word, i.e. include the space before it
      SCREAMING_SNAKE_CASE_									=						AddedToken(_A						,		lstrip=_A						,		rstrip=_A) if isinstance(_A						,		_A) else mask_token

      SCREAMING_SNAKE_CASE_									=						{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=_A						,		remove_space=_A						,		keep_accents=_A						,		bos_token=_A						,		eos_token=_A						,		unk_token=_A						,		sep_token=_A						,		pad_token=_A						,		cls_token=_A						,		mask_token=_A						,		additional_special_tokens=_A						,		sp_model_kwargs=self.sp_model_kwargs						,		**_A						,		)

      SCREAMING_SNAKE_CASE_									=						3

      SCREAMING_SNAKE_CASE_									=						do_lower_case
      SCREAMING_SNAKE_CASE_									=						remove_space
      SCREAMING_SNAKE_CASE_									=						keep_accents
      SCREAMING_SNAKE_CASE_									=						vocab_file

      SCREAMING_SNAKE_CASE_									=						spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(_A)

  @property
  def 						lowerCAmelCase__     (					self):
      return len(self.sp_model)

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						{self.convert_ids_to_tokens(_A): i for i in range(self.vocab_size)}
      vocab.update(self.added_tokens_encoder)
      return vocab

  def __getstate__(					self):
      SCREAMING_SNAKE_CASE_									=						self.__dict__.copy()
      SCREAMING_SNAKE_CASE_									=						None
      return state

  def __setstate__(					self						,		_A):
      SCREAMING_SNAKE_CASE_									=						d

      # for backward compatibility
      if not hasattr(self						,		'sp_model_kwargs'):
          SCREAMING_SNAKE_CASE_									=						{}

      SCREAMING_SNAKE_CASE_									=						spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(self.vocab_file)

  def 						lowerCAmelCase__     (					self						,		_A):
      if self.remove_space:
          SCREAMING_SNAKE_CASE_									=						' '.join(inputs.strip().split())
      else:
          SCREAMING_SNAKE_CASE_									=						inputs
      SCREAMING_SNAKE_CASE_									=						outputs.replace('``'						,		'"').replace('\'\''						,		'"')

      if not self.keep_accents:
          SCREAMING_SNAKE_CASE_									=						unicodedata.normalize('NFKD'						,		_A)
          SCREAMING_SNAKE_CASE_									=						''.join([c for c in outputs if not unicodedata.combining(_A)])
      if self.do_lower_case:
          SCREAMING_SNAKE_CASE_									=						outputs.lower()

      return outputs

  def 						lowerCAmelCase__     (					self						,		_A):
      SCREAMING_SNAKE_CASE_									=						self.preprocess_text(_A)
      SCREAMING_SNAKE_CASE_									=						self.sp_model.encode(_A						,		out_type=_A)
      SCREAMING_SNAKE_CASE_									=						[]
      for piece in pieces:
          if len(_A) > 1 and piece[-1] == str(',') and piece[-2].isdigit():
              SCREAMING_SNAKE_CASE_									=						self.sp_model.EncodeAsPieces(piece[:-1].replace(_A						,		''))
              if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                  if len(cur_pieces[0]) == 1:
                      SCREAMING_SNAKE_CASE_									=						cur_pieces[1:]
                  else:
                      SCREAMING_SNAKE_CASE_									=						cur_pieces[0][1:]
              cur_pieces.append(piece[-1])
              new_pieces.extend(_A)
          else:
              new_pieces.append(_A)

      return new_pieces

  def 						lowerCAmelCase__     (					self						,		_A):
      return self.sp_model.PieceToId(_A)

  def 						lowerCAmelCase__     (					self						,		_A):
      return self.sp_model.IdToPiece(_A)

  def 						lowerCAmelCase__     (					self						,		_A):
      SCREAMING_SNAKE_CASE_									=						''.join(_A).replace(_A						,		' ').strip()
      return out_string

  def 						lowerCAmelCase__     (					self						,		_A						,		_A = False						,		_A = None						,		_A = True						,		**_A						,		):
      SCREAMING_SNAKE_CASE_									=						kwargs.pop('use_source_tokenizer'						,		_A)

      SCREAMING_SNAKE_CASE_									=						self.convert_ids_to_tokens(_A						,		skip_special_tokens=_A)

      # To avoid mixing byte-level and unicode for byte-level BPT
      # we need to build string separately for added tokens and byte-level tokens
      # cf. https://github.com/huggingface/transformers/issues/1133
      SCREAMING_SNAKE_CASE_									=						[]
      SCREAMING_SNAKE_CASE_									=						[]
      for token in filtered_tokens:
          if skip_special_tokens and token in self.all_special_ids:
              continue
          if token in self.added_tokens_encoder:
              if current_sub_text:
                  sub_texts.append(self.convert_tokens_to_string(_A))
                  SCREAMING_SNAKE_CASE_									=						[]
              sub_texts.append(_A)
          else:
              current_sub_text.append(_A)
      if current_sub_text:
          sub_texts.append(self.convert_tokens_to_string(_A))

      # Mimic the behavior of the Rust tokenizer:
      # By default, there are no spaces between special tokens
      SCREAMING_SNAKE_CASE_									=						''.join(_A)

      SCREAMING_SNAKE_CASE_									=						(
          clean_up_tokenization_spaces
          if clean_up_tokenization_spaces is not None
          else self.clean_up_tokenization_spaces
      )
      if clean_up_tokenization_spaces:
          SCREAMING_SNAKE_CASE_									=						self.clean_up_tokenization(_A)
          return clean_text
      else:
          return text

  def 						lowerCAmelCase__     (					self						,		_A						,		_A = None):
      SCREAMING_SNAKE_CASE_									=						[self.sep_token_id]
      SCREAMING_SNAKE_CASE_									=						[self.cls_token_id]
      if token_ids_a is None:
          return token_ids_a + sep + cls
      return token_ids_a + sep + token_ids_a + sep + cls

  def 						lowerCAmelCase__     (					self						,		_A						,		_A = None						,		_A = False):

      if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=_A						,		token_ids_a=_A						,		already_has_special_tokens=_A)

      if token_ids_a is not None:
          return ([0] * len(_A)) + [1] + ([0] * len(_A)) + [1, 1]
      return ([0] * len(_A)) + [1, 1]

  def 						lowerCAmelCase__     (					self						,		_A						,		_A = None):
      SCREAMING_SNAKE_CASE_									=						[self.sep_token_id]
      SCREAMING_SNAKE_CASE_									=						[2]

      if token_ids_a is None:
          return len(token_ids_a + sep) * [0] + cls_segment_id
      return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1] + cls_segment_id


  def 						lowerCAmelCase__     (					self						,		_A						,		_A = None):
      if not os.path.isdir(_A):
          logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
          return
      SCREAMING_SNAKE_CASE_									=						os.path.join(
          _A						,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

      if os.path.abspath(self.vocab_file) != os.path.abspath(_A) and os.path.isfile(self.vocab_file):
          copyfile(self.vocab_file						,		_A)
      elif not os.path.isfile(self.vocab_file):
          with open(_A						,		'wb') as fi:
              SCREAMING_SNAKE_CASE_									=						self.sp_model.serialized_model_proto()
              fi.write(_A)

      return (out_vocab_file,)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__  :      str			=    logging.get_logger(__name__)

UpperCamelCase__  :      Optional[int]			=    {
    "facebook/dpr-ctx_encoder-single-nq-base": (
        "https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/config.json"
    ),
    "facebook/dpr-question_encoder-single-nq-base": (
        "https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/config.json"
    ),
    "facebook/dpr-reader-single-nq-base": (
        "https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/config.json"
    ),
    "facebook/dpr-ctx_encoder-multiset-base": (
        "https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/config.json"
    ),
    "facebook/dpr-question_encoder-multiset-base": (
        "https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/config.json"
    ),
    "facebook/dpr-reader-multiset-base": (
        "https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/config.json"
    ),
}



class       __snake_case  (    lowerCAmelCase__       ):
  __lowerCAmelCase     :	Optional[int]						 =  'dpr'

  def __init__(					self						,		_A=30522						,		_A=768						,		_A=12						,		_A=12						,		_A=3072						,		_A="gelu"						,		_A=0.1						,		_A=0.1						,		_A=512						,		_A=2						,		_A=0.0_2						,		_A=1E-12						,		_A=0						,		_A="absolute"						,		_A = 0						,		**_A						,		):
      super().__init__(pad_token_id=_A						,		**_A)

      SCREAMING_SNAKE_CASE_									=						vocab_size
      SCREAMING_SNAKE_CASE_									=						hidden_size
      SCREAMING_SNAKE_CASE_									=						num_hidden_layers
      SCREAMING_SNAKE_CASE_									=						num_attention_heads
      SCREAMING_SNAKE_CASE_									=						hidden_act
      SCREAMING_SNAKE_CASE_									=						intermediate_size
      SCREAMING_SNAKE_CASE_									=						hidden_dropout_prob
      SCREAMING_SNAKE_CASE_									=						attention_probs_dropout_prob
      SCREAMING_SNAKE_CASE_									=						max_position_embeddings
      SCREAMING_SNAKE_CASE_									=						type_vocab_size
      SCREAMING_SNAKE_CASE_									=						initializer_range
      SCREAMING_SNAKE_CASE_									=						layer_norm_eps
      SCREAMING_SNAKE_CASE_									=						projection_dim
      SCREAMING_SNAKE_CASE_									=						position_embedding_type


