



from typing import Dict, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import flip_channel_order, resize, to_channel_dimension_format, to_pil_image
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_pytesseract_available, is_vision_available, logging, requires_backends


if is_vision_available():
  import PIL

# soft dependency
if is_pytesseract_available():
  import pytesseract

a_			=					logging.get_logger(__name__)





def 		a__			(		_UpperCamelCase			: Tuple ,_UpperCamelCase			: Dict ,_UpperCamelCase			: Optional[Any]							):
   return [
       int(10_00 * (box[0] / width)							),
       int(10_00 * (box[1] / height)							),
       int(10_00 * (box[2] / width)							),
       int(10_00 * (box[3] / height)							),
   ]





def 		a__			(		_UpperCamelCase			: np.ndarray ,_UpperCamelCase			: Optional[str] ,_UpperCamelCase			: Optional[str] = None							):
   __lowerCamelCase													=    tesseract_config if tesseract_config is not None else ''''''

   # apply OCR
   __lowerCamelCase													=    to_pil_image(_UpperCamelCase							)
   __lowerCamelCase							,__lowerCamelCase													=    pil_image.size
   __lowerCamelCase													=    pytesseract.image_to_data(_UpperCamelCase ,lang=_UpperCamelCase ,output_type='''dict''' ,config=_UpperCamelCase							)
   __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    data['''text'''], data['''left'''], data['''top'''], data['''width'''], data['''height''']

   # filter empty words and corresponding coordinates
   __lowerCamelCase													=    [idx for idx, word in enumerate(_UpperCamelCase							) if not word.strip()]
   __lowerCamelCase													=    [word for idx, word in enumerate(_UpperCamelCase							) if idx not in irrelevant_indices]
   __lowerCamelCase													=    [coord for idx, coord in enumerate(_UpperCamelCase							) if idx not in irrelevant_indices]
   __lowerCamelCase													=    [coord for idx, coord in enumerate(_UpperCamelCase							) if idx not in irrelevant_indices]
   __lowerCamelCase													=    [coord for idx, coord in enumerate(_UpperCamelCase							) if idx not in irrelevant_indices]
   __lowerCamelCase													=    [coord for idx, coord in enumerate(_UpperCamelCase							) if idx not in irrelevant_indices]

   # turn coordinates into (left, top, left+width, top+height) format
   __lowerCamelCase													=    []
   for x, y, w, h in zip(_UpperCamelCase ,_UpperCamelCase ,_UpperCamelCase ,_UpperCamelCase							):
      __lowerCamelCase													=    [x, y, x + w, y + h]
      actual_boxes.append(_UpperCamelCase							)

   # finally, normalize the bounding boxes
   __lowerCamelCase													=    []
   for box in actual_boxes:
      normalized_boxes.append(normalize_box(_UpperCamelCase ,_UpperCamelCase ,_UpperCamelCase							)							)

   assert len(_UpperCamelCase							) == len(_UpperCamelCase							), "Not as many words as there are bounding boxes"

   return words, normalized_boxes





class  __lowerCAmelCase     (	lowerCAmelCase__   ):
 lowerCAmelCase__       					=  ["""pixel_values"""]




 def __init__(						self	,				__UpperCAmelCase = True	,				__UpperCAmelCase = None	,				__UpperCAmelCase = PILImageResampling.BILINEAR	,				__UpperCAmelCase = True	,				__UpperCAmelCase = None	,				__UpperCAmelCase = ""	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    super().__init__(**__UpperCAmelCase				)
    __lowerCamelCase													=    size if size is not None else {'''height''': 224, '''width''': 224}
    __lowerCamelCase													=    get_size_dict(__UpperCAmelCase				)

    __lowerCamelCase													=    do_resize
    __lowerCamelCase													=    size
    __lowerCamelCase													=    resample
    __lowerCamelCase													=    apply_ocr
    __lowerCamelCase													=    ocr_lang
    __lowerCamelCase													=    tesseract_config




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase = PILImageResampling.BILINEAR	,				__UpperCAmelCase = None	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    __lowerCamelCase													=    get_size_dict(__UpperCAmelCase				)
    if "height" not in size or "width" not in size:
       raise ValueError(F"""The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}"""				)
    __lowerCamelCase													=    (size['''height'''], size['''width'''])
    return resize(__UpperCAmelCase	,				size=__UpperCAmelCase	,				resample=__UpperCAmelCase	,				data_format=__UpperCAmelCase	,				**__UpperCAmelCase				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None	,				__UpperCAmelCase = None	,				__UpperCAmelCase = None	,				__UpperCAmelCase = None	,				__UpperCAmelCase = None	,				__UpperCAmelCase = None	,				__UpperCAmelCase = None	,				__UpperCAmelCase = ChannelDimension.FIRST	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    __lowerCamelCase													=    do_resize if do_resize is not None else self.do_resize
    __lowerCamelCase													=    size if size is not None else self.size
    __lowerCamelCase													=    get_size_dict(__UpperCAmelCase				)
    __lowerCamelCase													=    resample if resample is not None else self.resample
    __lowerCamelCase													=    apply_ocr if apply_ocr is not None else self.apply_ocr
    __lowerCamelCase													=    ocr_lang if ocr_lang is not None else self.ocr_lang
    __lowerCamelCase													=    tesseract_config if tesseract_config is not None else self.tesseract_config

    __lowerCamelCase													=    make_list_of_images(__UpperCAmelCase				)

    if not valid_images(__UpperCAmelCase				):
       raise ValueError(
           '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
           '''torch.Tensor, tf.Tensor or jax.ndarray.'''				)

    if do_resize and size is None:
       raise ValueError('''Size must be specified if do_resize is True.'''				)

    # All transformations expect numpy arrays.
    __lowerCamelCase													=    [to_numpy_array(__UpperCAmelCase				) for image in images]

    if apply_ocr:
       requires_backends(self	,				'''pytesseract'''				)
       __lowerCamelCase													=    []
       __lowerCamelCase													=    []
       for image in images:
          __lowerCamelCase							,__lowerCamelCase													=    apply_tesseract(__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase				)
          words_batch.append(__UpperCAmelCase				)
          boxes_batch.append(__UpperCAmelCase				)

    if do_resize:
       __lowerCamelCase													=    [self.resize(image=__UpperCAmelCase	,				size=__UpperCAmelCase	,				resample=__UpperCAmelCase				) for image in images]

    # flip color channels from RGB to BGR (as Detectron2 requires this)
    __lowerCamelCase													=    [flip_channel_order(__UpperCAmelCase				) for image in images]
    __lowerCamelCase													=    [to_channel_dimension_format(__UpperCAmelCase	,				__UpperCAmelCase				) for image in images]

    __lowerCamelCase													=    BatchFeature(data={'''pixel_values''': images}	,				tensor_type=__UpperCAmelCase				)

    if apply_ocr:
       __lowerCamelCase													=    words_batch
       __lowerCamelCase													=    boxes_batch
    return data











import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_			=					logging.get_logger(__name__)
a_			=					{"""vocab_file""": """spiece.model"""}

a_			=					{
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

a_			=					{
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

a_			=					"""▁"""



class  __lowerCAmelCase     (	lowerCAmelCase__   ):
 lowerCAmelCase__       					=  VOCAB_FILES_NAMES
 lowerCAmelCase__       					=  PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       					=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




 def __init__(						self	,				__UpperCAmelCase	,				__UpperCAmelCase=True	,				__UpperCAmelCase=True	,				__UpperCAmelCase=False	,				__UpperCAmelCase="[CLS]"	,				__UpperCAmelCase="[SEP]"	,				__UpperCAmelCase="<unk>"	,				__UpperCAmelCase="[SEP]"	,				__UpperCAmelCase="<pad>"	,				__UpperCAmelCase="[CLS]"	,				__UpperCAmelCase="[MASK]"	,				__UpperCAmelCase = None	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    # Mask token behave like a normal word, i.e. include the space before it and
    # is included in the raw text, there should be a match in a non-normalized sentence.
    __lowerCamelCase													=    (
        AddedToken(__UpperCAmelCase	,				lstrip=__UpperCAmelCase	,				rstrip=__UpperCAmelCase	,				normalized=__UpperCAmelCase				)
        if isinstance(__UpperCAmelCase	,				__UpperCAmelCase				)
        else mask_token
    )

    __lowerCamelCase													=    {} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        do_lower_case=__UpperCAmelCase	,				remove_space=__UpperCAmelCase	,				keep_accents=__UpperCAmelCase	,				bos_token=__UpperCAmelCase	,				eos_token=__UpperCAmelCase	,				unk_token=__UpperCAmelCase	,				sep_token=__UpperCAmelCase	,				pad_token=__UpperCAmelCase	,				cls_token=__UpperCAmelCase	,				mask_token=__UpperCAmelCase	,				sp_model_kwargs=self.sp_model_kwargs	,				**__UpperCAmelCase	,				)

    __lowerCamelCase													=    do_lower_case
    __lowerCamelCase													=    remove_space
    __lowerCamelCase													=    keep_accents
    __lowerCamelCase													=    vocab_file

    __lowerCamelCase													=    spm.SentencePieceProcessor(**self.sp_model_kwargs				)
    self.sp_model.Load(__UpperCAmelCase				)




 @property
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return len(self.sp_model				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    {self.convert_ids_to_tokens(__UpperCAmelCase				): i for i in range(self.vocab_size				)}
    vocab.update(self.added_tokens_encoder				)
    return vocab




 def __getstate__(						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.__dict__.copy()
    __lowerCamelCase													=    None
    return state




 def __setstate__(						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    d

    # for backward compatibility
    if not hasattr(self	,				'''sp_model_kwargs'''				):
       __lowerCamelCase													=    {}

    __lowerCamelCase													=    spm.SentencePieceProcessor(**self.sp_model_kwargs				)
    self.sp_model.Load(self.vocab_file				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    if self.remove_space:
       __lowerCamelCase													=    ''' '''.join(inputs.strip().split()				)
    else:
       __lowerCamelCase													=    inputs
    __lowerCamelCase													=    outputs.replace('''``'''	,				'''"'''				).replace('''\'\''''	,				'''"'''				)

    if not self.keep_accents:
       __lowerCamelCase													=    unicodedata.normalize('''NFKD'''	,				__UpperCAmelCase				)
       __lowerCamelCase													=    ''''''.join([c for c in outputs if not unicodedata.combining(__UpperCAmelCase				)]				)
    if self.do_lower_case:
       __lowerCamelCase													=    outputs.lower()

    return outputs




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    self.preprocess_text(__UpperCAmelCase				)
    __lowerCamelCase													=    self.sp_model.encode(__UpperCAmelCase	,				out_type=__UpperCAmelCase				)
    __lowerCamelCase													=    []
    for piece in pieces:
       if len(__UpperCAmelCase				) > 1 and piece[-1] == str(''','''				) and piece[-2].isdigit():
          __lowerCamelCase													=    self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCAmelCase	,				''''''				)				)
          if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
             if len(cur_pieces[0]				) == 1:
                __lowerCamelCase													=    cur_pieces[1:]
             else:
                __lowerCamelCase													=    cur_pieces[0][1:]
          cur_pieces.append(piece[-1]				)
          new_pieces.extend(__UpperCAmelCase				)
       else:
          new_pieces.append(__UpperCAmelCase				)

    return new_pieces




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    return self.sp_model.PieceToId(__UpperCAmelCase				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    return self.sp_model.IdToPiece(__UpperCAmelCase				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    []
    __lowerCamelCase													=    ''''''
    __lowerCamelCase													=    False
    for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
          if not prev_is_special:
             out_string += " "
          out_string += self.sp_model.decode(__UpperCAmelCase				) + token
          __lowerCamelCase													=    True
          __lowerCamelCase													=    []
       else:
          current_sub_tokens.append(__UpperCAmelCase				)
          __lowerCamelCase													=    False
    out_string += self.sp_model.decode(__UpperCAmelCase				)
    return out_string.strip()




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    __lowerCamelCase													=    [self.sep_token_id]
    __lowerCamelCase													=    [self.cls_token_id]
    if token_ids_a is None:
       return cls + token_ids_a + sep
    return cls + token_ids_a + sep + token_ids_a + sep




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None	,				__UpperCAmelCase = False				):



    '''simple docstring'''

    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__UpperCAmelCase	,				token_ids_a=__UpperCAmelCase	,				already_has_special_tokens=__UpperCAmelCase				)

    if token_ids_a is not None:
       return [1] + ([0] * len(__UpperCAmelCase				)) + [1] + ([0] * len(__UpperCAmelCase				)) + [1]
    return [1] + ([0] * len(__UpperCAmelCase				)) + [1]




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    __lowerCamelCase													=    [self.sep_token_id]
    __lowerCamelCase													=    [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep				) * [0]
    return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    if not os.path.isdir(__UpperCAmelCase				):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""				)
       return
    __lowerCamelCase													=    os.path.join(
        __UpperCAmelCase	,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

    if os.path.abspath(self.vocab_file				) != os.path.abspath(__UpperCAmelCase				) and os.path.isfile(self.vocab_file				):
       copyfile(self.vocab_file	,				__UpperCAmelCase				)
    elif not os.path.isfile(self.vocab_file				):
       with open(__UpperCAmelCase	,				'''wb'''				) as fi:
          __lowerCamelCase													=    self.sp_model.serialized_model_proto()
          fi.write(__UpperCAmelCase				)

    return (out_vocab_file,)






