





from copy import deepcopy
from typing import Optional, Union

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, is_tf_available, is_torch_available


if is_torch_available():
      import torch

if is_tf_available():
      import tensorflow as tf



class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       Dict      					=["""image_processor"""]
  a_							:       Optional[Any]      					="""SamImageProcessor"""







  def __init__(      self      :					Tuple      ,				UpperCamelCase      :					Any							):


    '''simple docstring'''





    super().__init__(UpperCamelCase							)
    _snake_case   :				Dict														=	self.image_processor
    _snake_case   :				str														=	-10
    _snake_case   :				int														=	self.image_processor.size['longest_edge']







  def __call__(      self      :					int      ,				UpperCamelCase      :					Any=None      ,				UpperCamelCase      :					Dict=None      ,				UpperCamelCase      :					Optional[Any]=None      ,				UpperCamelCase      :					List[Any]=None      ,				UpperCamelCase      :					Optional[Union[str, TensorType]] = None      ,				**UpperCamelCase      :					str      ,				):


    '''simple docstring'''





    _snake_case   :				Optional[int]														=	self.image_processor(
        UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				**UpperCamelCase      ,				)

    # pop arguments that are not used in the foward but used nevertheless
    _snake_case   :				int														=	encoding_image_processor['original_sizes']

    if hasattr(UpperCamelCase      ,				'numpy'							):  # Checks if Torch or TF tensor
      _snake_case   :				str														=	original_sizes.numpy()

    _snake_case							,					_snake_case							,					_snake_case   :				List[str]														=	self._check_and_preprocess_points(
        input_points=UpperCamelCase      ,				input_labels=UpperCamelCase      ,				input_boxes=UpperCamelCase      ,				)

    _snake_case   :				Optional[Any]														=	self._normalize_and_convert(
        UpperCamelCase      ,				UpperCamelCase      ,				input_points=UpperCamelCase      ,				input_labels=UpperCamelCase      ,				input_boxes=UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				)

    return encoding_image_processor







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					Tuple      ,				UpperCamelCase      :					Optional[int]      ,				UpperCamelCase      :					Tuple=None      ,				UpperCamelCase      :					Union[str, Any]=None      ,				UpperCamelCase      :					Tuple=None      ,				UpperCamelCase      :					int="pt"      ,				):


    '''simple docstring'''





    if input_points is not None:
      if len(UpperCamelCase							) != len(UpperCamelCase							):
        _snake_case   :				Dict														=	[
            self._normalize_coordinates(self.target_size      ,				UpperCamelCase      ,				original_sizes[0]							) for point in input_points
        ]
      else:
        _snake_case   :				Dict														=	[
            self._normalize_coordinates(self.target_size      ,				UpperCamelCase      ,				UpperCamelCase							)
            for point, original_size in zip(UpperCamelCase      ,				UpperCamelCase							)
        ]
      # check that all arrays have the same shape
      if not all(point.shape == input_points[0].shape for point in input_points							):
        if input_labels is not None:
          _snake_case							,					_snake_case   :				Optional[Any]														=	self._pad_points_and_labels(UpperCamelCase      ,				UpperCamelCase							)

      _snake_case   :				Tuple														=	np.array(UpperCamelCase							)

    if input_labels is not None:
      _snake_case   :				Optional[int]														=	np.array(UpperCamelCase							)

    if input_boxes is not None:
      if len(UpperCamelCase							) != len(UpperCamelCase							):
        _snake_case   :				Dict														=	[
            self._normalize_coordinates(self.target_size      ,				UpperCamelCase      ,				original_sizes[0]      ,				is_bounding_box=UpperCamelCase							)
            for box in input_boxes
        ]
      else:
        _snake_case   :				List[str]														=	[
            self._normalize_coordinates(self.target_size      ,				UpperCamelCase      ,				UpperCamelCase      ,				is_bounding_box=UpperCamelCase							)
            for box, original_size in zip(UpperCamelCase      ,				UpperCamelCase							)
        ]
      _snake_case   :				str														=	np.array(UpperCamelCase							)

    if input_boxes is not None:
      if return_tensors == "pt":
        _snake_case   :				int														=	torch.from_numpy(UpperCamelCase							)
        # boxes batch size of 1 by default
        _snake_case   :				Any														=	input_boxes.unsqueeze(1							) if len(input_boxes.shape							) != 3 else input_boxes
      elif return_tensors == "tf":
        _snake_case   :				List[str]														=	tf.convert_to_tensor(UpperCamelCase							)
        # boxes batch size of 1 by default
        _snake_case   :				Any														=	tf.expand_dims(UpperCamelCase      ,				1							) if len(input_boxes.shape							) != 3 else input_boxes
      encoding_image_processor.update({'input_boxes': input_boxes}							)
    if input_points is not None:
      if return_tensors == "pt":
        _snake_case   :				Optional[int]														=	torch.from_numpy(UpperCamelCase							)
        # point batch size of 1 by default
        _snake_case   :				Dict														=	input_points.unsqueeze(1							) if len(input_points.shape							) != 4 else input_points
      elif return_tensors == "tf":
        _snake_case   :				int														=	tf.convert_to_tensor(UpperCamelCase							)
        # point batch size of 1 by default
        _snake_case   :				Optional[Any]														=	tf.expand_dims(UpperCamelCase      ,				1							) if len(input_points.shape							) != 4 else input_points
      encoding_image_processor.update({'input_points': input_points}							)
    if input_labels is not None:
      if return_tensors == "pt":
        _snake_case   :				Optional[Any]														=	torch.from_numpy(UpperCamelCase							)
        # point batch size of 1 by default
        _snake_case   :				Optional[int]														=	input_labels.unsqueeze(1							) if len(input_labels.shape							) != 3 else input_labels
      elif return_tensors == "tf":
        _snake_case   :				int														=	tf.convert_to_tensor(UpperCamelCase							)
        # point batch size of 1 by default
        _snake_case   :				Tuple														=	tf.expand_dims(UpperCamelCase      ,				1							) if len(input_labels.shape							) != 3 else input_labels
      encoding_image_processor.update({'input_labels': input_labels}							)

    return encoding_image_processor







  def UpperCamelCase_ (      self      :					Optional[int]      ,				UpperCamelCase      :					Tuple      ,				UpperCamelCase      :					List[Any]							):


    '''simple docstring'''





    _snake_case   :				Any														=	max([point.shape[0] for point in input_points]							)
    _snake_case   :				int														=	[]
    for i, point in enumerate(UpperCamelCase							):
      if point.shape[0] != expected_nb_points:
        _snake_case   :				List[str]														=	np.concatenate(
            [point, np.zeros((expected_nb_points - point.shape[0], 2)							) + self.point_pad_value]      ,				axis=0							)
        _snake_case   :				Optional[Any]														=	np.append(input_labels[i]      ,				[self.point_pad_value]							)
      processed_input_points.append(UpperCamelCase							)
    _snake_case   :				Union[str, Any]														=	processed_input_points
    return input_points, input_labels







  def UpperCamelCase_ (      self      :					Optional[int]      ,				UpperCamelCase      :					int      ,				UpperCamelCase      :					np.ndarray      ,				UpperCamelCase      :					Dict      ,				UpperCamelCase      :					List[str]=False							):


    '''simple docstring'''





    _snake_case							,					_snake_case   :				Optional[int]														=	original_size
    _snake_case							,					_snake_case   :				str														=	self.image_processor._get_preprocess_shape(UpperCamelCase      ,				longest_edge=UpperCamelCase							)
    _snake_case   :				Optional[Any]														=	deepcopy(UpperCamelCase							).astype(UpperCamelCase							)

    if is_bounding_box:
      _snake_case   :				str														=	coords.reshape(-1      ,				2      ,				2							)

    _snake_case   :				Union[str, Any]														=	coords[..., 0] * (new_w / old_w)
    _snake_case   :				Dict														=	coords[..., 1] * (new_h / old_h)

    if is_bounding_box:
      _snake_case   :				Any														=	coords.reshape(-1      ,				4							)

    return coords







  def UpperCamelCase_ (      self      :					Any      ,				UpperCamelCase      :					Union[str, Any]=None      ,				UpperCamelCase      :					Union[str, Any]=None      ,				UpperCamelCase      :					int=None      ,				):


    '''simple docstring'''





    if input_points is not None:
      if hasattr(UpperCamelCase      ,				'numpy'							):  # Checks for TF or Torch tensor
        _snake_case   :				Union[str, Any]														=	input_points.numpy().tolist()

      if not isinstance(UpperCamelCase      ,				UpperCamelCase							) or not isinstance(input_points[0]      ,				UpperCamelCase							):
        raise ValueError('Input points must be a list of list of floating points.'							)
      _snake_case   :				Tuple														=	[np.array(UpperCamelCase							) for input_point in input_points]
    else:
      _snake_case   :				Optional[Any]														=	None

    if input_labels is not None:
      if hasattr(UpperCamelCase      ,				'numpy'							):
        _snake_case   :				Dict														=	input_labels.numpy().tolist()

      if not isinstance(UpperCamelCase      ,				UpperCamelCase							) or not isinstance(input_labels[0]      ,				UpperCamelCase							):
        raise ValueError('Input labels must be a list of list integers.'							)
      _snake_case   :				Optional[Any]														=	[np.array(UpperCamelCase							) for label in input_labels]
    else:
      _snake_case   :				Optional[Any]														=	None

    if input_boxes is not None:
      if hasattr(UpperCamelCase      ,				'numpy'							):
        _snake_case   :				Tuple														=	input_boxes.numpy().tolist()

      if (
          not isinstance(UpperCamelCase      ,				UpperCamelCase							)
          or not isinstance(input_boxes[0]      ,				UpperCamelCase							)
          or not isinstance(input_boxes[0][0]      ,				UpperCamelCase							)
      ):
        raise ValueError('Input boxes must be a list of list of list of floating points.'							)
      _snake_case   :				Tuple														=	[np.array(UpperCamelCase							).astype(np.floataa							) for box in input_boxes]
    else:
      _snake_case   :				Any														=	None

    return input_points, input_labels, input_boxes







  @property
  def UpperCamelCase_ (      self      :					List[Any]							):


    '''simple docstring'''





    _snake_case   :				int														=	self.image_processor.model_input_names
    return list(dict.fromkeys(UpperCamelCase							)							)







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				*UpperCamelCase      :					Any      ,				**UpperCamelCase      :					Any							):


    '''simple docstring'''





    return self.image_processor.post_process_masks(*UpperCamelCase      ,				**UpperCamelCase							)











import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCAmelCase_						   =    logging.get_logger(__name__)

lowerCAmelCase_						   =    {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase_						   =    {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase_						   =    {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

lowerCAmelCase_						   =    {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}



class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       Union[str, Any]      					=VOCAB_FILES_NAMES
  a_							:       List[str]      					=PRETRAINED_VOCAB_FILES_MAP
  a_							:       str      					=PRETRAINED_INIT_CONFIGURATION
  a_							:       Optional[Any]      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_							:       List[Any]      					=RealmTokenizer







  def __init__(      self      :					List[str]      ,				UpperCamelCase      :					Optional[int]=None      ,				UpperCamelCase      :					List[Any]=None      ,				UpperCamelCase      :					Optional[Any]=True      ,				UpperCamelCase      :					Optional[Any]="[UNK]"      ,				UpperCamelCase      :					Any="[SEP]"      ,				UpperCamelCase      :					Optional[Any]="[PAD]"      ,				UpperCamelCase      :					Optional[int]="[CLS]"      ,				UpperCamelCase      :					Optional[Any]="[MASK]"      ,				UpperCamelCase      :					Dict=True      ,				UpperCamelCase      :					Optional[int]=None      ,				**UpperCamelCase      :					Union[str, Any]      ,				):


    '''simple docstring'''





    super().__init__(
        UpperCamelCase      ,				tokenizer_file=UpperCamelCase      ,				do_lower_case=UpperCamelCase      ,				unk_token=UpperCamelCase      ,				sep_token=UpperCamelCase      ,				pad_token=UpperCamelCase      ,				cls_token=UpperCamelCase      ,				mask_token=UpperCamelCase      ,				tokenize_chinese_chars=UpperCamelCase      ,				strip_accents=UpperCamelCase      ,				**UpperCamelCase      ,				)

    _snake_case   :				int														=	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
    if (
        normalizer_state.get('lowercase'      ,				UpperCamelCase							) != do_lower_case
        or normalizer_state.get('strip_accents'      ,				UpperCamelCase							) != strip_accents
        or normalizer_state.get('handle_chinese_chars'      ,				UpperCamelCase							) != tokenize_chinese_chars
    ):
      _snake_case   :				int														=	getattr(UpperCamelCase      ,				normalizer_state.pop('type'							)							)
      _snake_case   :				List[str]														=	do_lower_case
      _snake_case   :				List[Any]														=	strip_accents
      _snake_case   :				Dict														=	tokenize_chinese_chars
      _snake_case   :				Any														=	normalizer_class(**UpperCamelCase							)

    _snake_case   :				Optional[int]														=	do_lower_case







  def UpperCamelCase_ (      self      :					Dict      ,				UpperCamelCase      :					Any      ,				**UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				Tuple														=	PaddingStrategy.MAX_LENGTH

    _snake_case   :				Any														=	text
    _snake_case   :				List[str]														=	kwargs.pop('text_pair'      ,				UpperCamelCase							)
    _snake_case   :				int														=	kwargs.pop('return_tensors'      ,				UpperCamelCase							)

    _snake_case   :				Optional[int]														=	{
        'input_ids': [],
        'attention_mask': [],
        'token_type_ids': [],
    }

    for idx, candidate_text in enumerate(UpperCamelCase							):
      if batch_text_pair is not None:
        _snake_case   :				List[Any]														=	batch_text_pair[idx]
      else:
        _snake_case   :				Optional[Any]														=	None

      _snake_case   :				Optional[int]														=	super().__call__(UpperCamelCase      ,				UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				**UpperCamelCase							)

      _snake_case   :				str														=	encoded_candidates.get('input_ids'							)
      _snake_case   :				Tuple														=	encoded_candidates.get('attention_mask'							)
      _snake_case   :				List[str]														=	encoded_candidates.get('token_type_ids'							)

      if encoded_input_ids is not None:
        output_data["input_ids"].append(UpperCamelCase							)
      if encoded_attention_mask is not None:
        output_data["attention_mask"].append(UpperCamelCase							)
      if encoded_token_type_ids is not None:
        output_data["token_type_ids"].append(UpperCamelCase							)

    _snake_case   :				str														=	{key: item for key, item in output_data.items() if len(UpperCamelCase							) != 0}

    return BatchEncoding(UpperCamelCase      ,				tensor_type=UpperCamelCase							)







  def UpperCamelCase_ (      self      :					str      ,				UpperCamelCase      :					Optional[Any]      ,				UpperCamelCase      :					Union[str, Any]=None							):


    '''simple docstring'''





    _snake_case   :				Dict														=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
      output += token_ids_a + [self.sep_token_id]

    return output







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					List[int]      ,				UpperCamelCase      :					Optional[List[int]] = None							):


    '''simple docstring'''





    _snake_case   :				int														=	[self.sep_token_id]
    _snake_case   :				Dict														=	[self.cls_token_id]
    if token_ids_a is None:
      return len(cls + token_ids_a + sep							) * [0]
    return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					str      ,				UpperCamelCase      :					Optional[str] = None							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	self._tokenizer.model.save(UpperCamelCase      ,				name=UpperCamelCase							)
    return tuple(UpperCamelCase							)




