





import re
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin



class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       Dict      					=["""image_processor""", """tokenizer"""]
  a_							:       Dict      					="""AutoImageProcessor"""
  a_							:       str      					="""AutoTokenizer"""







  def __init__(      self      :					Optional[Any]      ,				UpperCamelCase      :					List[str]=None      ,				UpperCamelCase      :					int=None      ,				**UpperCamelCase      :					Any							):


    '''simple docstring'''





    _snake_case   :				Any														=	None
    if "feature_extractor" in kwargs:
      warnings.warn(
          'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
          ' instead.'      ,				UpperCamelCase      ,				)
      _snake_case   :				int														=	kwargs.pop('feature_extractor'							)

    _snake_case   :				Tuple														=	image_processor if image_processor is not None else feature_extractor
    if image_processor is None:
      raise ValueError('You need to specify an `image_processor`.'							)
    if tokenizer is None:
      raise ValueError('You need to specify a `tokenizer`.'							)

    super().__init__(UpperCamelCase      ,				UpperCamelCase							)
    _snake_case   :				Tuple														=	self.image_processor
    _snake_case   :				Optional[Any]														=	False







  def __call__(      self      :					str      ,				*UpperCamelCase      :					Tuple      ,				**UpperCamelCase      :					Optional[Any]							):


    '''simple docstring'''





    if self._in_target_context_manager:
      return self.current_processor(*UpperCamelCase      ,				**UpperCamelCase							)

    _snake_case   :				str														=	kwargs.pop('images'      ,				UpperCamelCase							)
    _snake_case   :				Optional[int]														=	kwargs.pop('text'      ,				UpperCamelCase							)
    if len(UpperCamelCase							) > 0:
      _snake_case   :				Optional[Any]														=	args[0]
      _snake_case   :				Dict														=	args[1:]

    if images is None and text is None:
      raise ValueError('You need to specify either an `images` or `text` input to process.'							)

    if images is not None:
      _snake_case   :				Dict														=	self.image_processor(UpperCamelCase      ,				*UpperCamelCase      ,				**UpperCamelCase							)
    if text is not None:
      _snake_case   :				Optional[Any]														=	self.tokenizer(UpperCamelCase      ,				**UpperCamelCase							)

    if text is None:
      return inputs
    elif images is None:
      return encodings
    else:
      _snake_case   :				Optional[Any]														=	encodings['input_ids']
      return inputs







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				*UpperCamelCase      :					List[str]      ,				**UpperCamelCase      :					Tuple							):


    '''simple docstring'''





    return self.tokenizer.batch_decode(*UpperCamelCase      ,				**UpperCamelCase							)







  def UpperCamelCase_ (      self      :					List[str]      ,				*UpperCamelCase      :					Any      ,				**UpperCamelCase      :					Optional[Any]							):


    '''simple docstring'''





    return self.tokenizer.decode(*UpperCamelCase      ,				**UpperCamelCase							)







  @contextmanager
  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    warnings.warn(
        '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
        'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
        'your images inputs, or in a separate call.'							)
    _snake_case   :				Any														=	True
    _snake_case   :				List[str]														=	self.tokenizer
    yield
    _snake_case   :				Optional[int]														=	self.image_processor
    _snake_case   :				Optional[int]														=	False







  def UpperCamelCase_ (      self      :					List[str]      ,				UpperCamelCase      :					Tuple      ,				UpperCamelCase      :					List[str]=False      ,				UpperCamelCase      :					Tuple=None							):


    '''simple docstring'''





    if added_vocab is None:
      _snake_case   :				int														=	self.tokenizer.get_added_vocab()

    _snake_case   :				Optional[int]														=	{}

    while tokens:
      _snake_case   :				Optional[Any]														=	re.search(R'<s_(.*?)>'      ,				UpperCamelCase      ,				re.IGNORECASE							)
      if start_token is None:
        break
      _snake_case   :				Tuple														=	start_token.group(1							)
      _snake_case   :				str														=	re.search(Rf"""</s_{key}>"""      ,				UpperCamelCase      ,				re.IGNORECASE							)
      _snake_case   :				int														=	start_token.group()
      if end_token is None:
        _snake_case   :				str														=	tokens.replace(UpperCamelCase      ,				''							)
      else:
        _snake_case   :				Optional[Any]														=	end_token.group()
        _snake_case   :				Tuple														=	re.escape(UpperCamelCase							)
        _snake_case   :				Any														=	re.escape(UpperCamelCase							)
        _snake_case   :				Union[str, Any]														=	re.search(f"""{start_token_escaped}(.*?){end_token_escaped}"""      ,				UpperCamelCase      ,				re.IGNORECASE							)
        if content is not None:
          _snake_case   :				Union[str, Any]														=	content.group(1							).strip()
          if r"<s_" in content and r"</s_" in content:  # non-leaf node
            _snake_case   :				Optional[int]														=	self.tokenajson(UpperCamelCase      ,				is_inner_value=UpperCamelCase      ,				added_vocab=UpperCamelCase							)
            if value:
              if len(UpperCamelCase							) == 1:
                _snake_case   :				List[Any]														=	value[0]
              _snake_case   :				Optional[Any]														=	value
          else:  # leaf nodes
            _snake_case   :				int														=	[]
            for leaf in content.split(R'<sep/>'							):
              _snake_case   :				Tuple														=	leaf.strip()
              if leaf in added_vocab and leaf[0] == "<" and leaf[-2:] == "/>":
                _snake_case   :				List[Any]														=	leaf[1:-2]  # for categorical special tokens
              output[key].append(UpperCamelCase							)
            if len(output[key]							) == 1:
              _snake_case   :				Optional[Any]														=	output[key][0]

        _snake_case   :				str														=	tokens[tokens.find(UpperCamelCase							) + len(UpperCamelCase							) :].strip()
        if tokens[:6] == r"<sep/>":  # non-leaf nodes
          return [output] + self.tokenajson(tokens[6:]      ,				is_inner_value=UpperCamelCase      ,				added_vocab=UpperCamelCase							)

    if len(UpperCamelCase							):
      return [output] if is_inner_value else output
    else:
      return [] if is_inner_value else {"text_sequence": tokens}







  @property
  def UpperCamelCase_ (      self      :					str							):


    '''simple docstring'''





    warnings.warn(
        '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'      ,				UpperCamelCase      ,				)
    return self.image_processor_class







  @property
  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    warnings.warn(
        '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'      ,				UpperCamelCase      ,				)
    return self.image_processor











import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCAmelCase_						   =    logging.get_logger(__name__)

lowerCAmelCase_						   =    {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase_						   =    {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase_						   =    {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

lowerCAmelCase_						   =    {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}



class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       Union[str, Any]      					=VOCAB_FILES_NAMES
  a_							:       List[str]      					=PRETRAINED_VOCAB_FILES_MAP
  a_							:       str      					=PRETRAINED_INIT_CONFIGURATION
  a_							:       Optional[Any]      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_							:       List[Any]      					=RealmTokenizer







  def __init__(      self      :					List[str]      ,				UpperCamelCase      :					Optional[int]=None      ,				UpperCamelCase      :					List[Any]=None      ,				UpperCamelCase      :					Optional[Any]=True      ,				UpperCamelCase      :					Optional[Any]="[UNK]"      ,				UpperCamelCase      :					Any="[SEP]"      ,				UpperCamelCase      :					Optional[Any]="[PAD]"      ,				UpperCamelCase      :					Optional[int]="[CLS]"      ,				UpperCamelCase      :					Optional[Any]="[MASK]"      ,				UpperCamelCase      :					Dict=True      ,				UpperCamelCase      :					Optional[int]=None      ,				**UpperCamelCase      :					Union[str, Any]      ,				):


    '''simple docstring'''





    super().__init__(
        UpperCamelCase      ,				tokenizer_file=UpperCamelCase      ,				do_lower_case=UpperCamelCase      ,				unk_token=UpperCamelCase      ,				sep_token=UpperCamelCase      ,				pad_token=UpperCamelCase      ,				cls_token=UpperCamelCase      ,				mask_token=UpperCamelCase      ,				tokenize_chinese_chars=UpperCamelCase      ,				strip_accents=UpperCamelCase      ,				**UpperCamelCase      ,				)

    _snake_case   :				int														=	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
    if (
        normalizer_state.get('lowercase'      ,				UpperCamelCase							) != do_lower_case
        or normalizer_state.get('strip_accents'      ,				UpperCamelCase							) != strip_accents
        or normalizer_state.get('handle_chinese_chars'      ,				UpperCamelCase							) != tokenize_chinese_chars
    ):
      _snake_case   :				int														=	getattr(UpperCamelCase      ,				normalizer_state.pop('type'							)							)
      _snake_case   :				List[str]														=	do_lower_case
      _snake_case   :				List[Any]														=	strip_accents
      _snake_case   :				Dict														=	tokenize_chinese_chars
      _snake_case   :				Any														=	normalizer_class(**UpperCamelCase							)

    _snake_case   :				Optional[int]														=	do_lower_case







  def UpperCamelCase_ (      self      :					Dict      ,				UpperCamelCase      :					Any      ,				**UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				Tuple														=	PaddingStrategy.MAX_LENGTH

    _snake_case   :				Any														=	text
    _snake_case   :				List[str]														=	kwargs.pop('text_pair'      ,				UpperCamelCase							)
    _snake_case   :				int														=	kwargs.pop('return_tensors'      ,				UpperCamelCase							)

    _snake_case   :				Optional[int]														=	{
        'input_ids': [],
        'attention_mask': [],
        'token_type_ids': [],
    }

    for idx, candidate_text in enumerate(UpperCamelCase							):
      if batch_text_pair is not None:
        _snake_case   :				List[Any]														=	batch_text_pair[idx]
      else:
        _snake_case   :				Optional[Any]														=	None

      _snake_case   :				Optional[int]														=	super().__call__(UpperCamelCase      ,				UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				**UpperCamelCase							)

      _snake_case   :				str														=	encoded_candidates.get('input_ids'							)
      _snake_case   :				Tuple														=	encoded_candidates.get('attention_mask'							)
      _snake_case   :				List[str]														=	encoded_candidates.get('token_type_ids'							)

      if encoded_input_ids is not None:
        output_data["input_ids"].append(UpperCamelCase							)
      if encoded_attention_mask is not None:
        output_data["attention_mask"].append(UpperCamelCase							)
      if encoded_token_type_ids is not None:
        output_data["token_type_ids"].append(UpperCamelCase							)

    _snake_case   :				str														=	{key: item for key, item in output_data.items() if len(UpperCamelCase							) != 0}

    return BatchEncoding(UpperCamelCase      ,				tensor_type=UpperCamelCase							)







  def UpperCamelCase_ (      self      :					str      ,				UpperCamelCase      :					Optional[Any]      ,				UpperCamelCase      :					Union[str, Any]=None							):


    '''simple docstring'''





    _snake_case   :				Dict														=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
      output += token_ids_a + [self.sep_token_id]

    return output







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					List[int]      ,				UpperCamelCase      :					Optional[List[int]] = None							):


    '''simple docstring'''





    _snake_case   :				int														=	[self.sep_token_id]
    _snake_case   :				Dict														=	[self.cls_token_id]
    if token_ids_a is None:
      return len(cls + token_ids_a + sep							) * [0]
    return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					str      ,				UpperCamelCase      :					Optional[str] = None							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	self._tokenizer.model.save(UpperCamelCase      ,				name=UpperCamelCase							)
    return tuple(UpperCamelCase							)




