


from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

a					  =     _symbol_database.Default()


a					  =     _descriptor_pool.Default().AddSerializedFile(
    b'\n\x19sentencepiece_model.proto\x12\rsentencepiece"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03'
)

a					  =     globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sentencepiece_model_pb2', _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
       a					  =     None
       a					  =     b'H\003'
       # (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
       # _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
       # _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
       # _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
       # _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
       a					  =     4_5
       a					  =     1_5_8_1
       a					  =     1_5_1_7
       a					  =     1_5_7_0
       a					  =     1_5_8_4
       a					  =     1_7_9_3
       a					  =     1_7_9_5
       a					  =     1_9_1_6
       a					  =     1_8_6_4
       a					  =     1_9_0_5
       a					  =     1_9_1_9
       a					  =     2_4_2_9
       a					  =     2_2_0_8
       a					  =     2_4_1_8
       a					  =     2_3_2_3
       a					  =     2_4_0_7
# @@protoc_insertion_point(module_scope)






from typing import Dict, Iterable, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


a					  =     logging.get_logger(__name__)







class        UpperCamelCase__   (	__magic_name__   ):
					__SCREAMING_SNAKE_CASE       :  List[Any]						=   ['pixel_values']

					def __init__(     self       :   List[str]							,					UpperCamelCase__       :   bool = True							,					UpperCamelCase__       :   Dict[str, int] = None							,					UpperCamelCase__       :   PILImageResampling = PILImageResampling.BICUBIC							,					UpperCamelCase__       :   bool = True							,					UpperCamelCase__       :   Dict[str, int] = None							,					UpperCamelCase__       :   bool = True							,					UpperCamelCase__       :   Union[int, float] = 1 / 255							,					UpperCamelCase__       :   bool = True							,					UpperCamelCase__       :   Optional[Union[float, Iterable[float]]] = IMAGENET_DEFAULT_MEAN							,					UpperCamelCase__       :   Optional[Union[float, Iterable[float]]] = IMAGENET_DEFAULT_STD							,					**UpperCamelCase__       :   Dict							,					):






										'''simple docstring'''






										super().__init__(**UpperCamelCase__       )
										lowercase_  							=			size if size is not None else {"""shortest_edge""": 224}
										lowercase_  							=			get_size_dict(UpperCamelCase__							,					default_to_square=UpperCamelCase__       )
										lowercase_  							=			crop_size if crop_size is not None else {"""height""": 224, """width""": 224}
										lowercase_  							=			get_size_dict(UpperCamelCase__							,					param_name="""crop_size"""       )

										lowercase_  							=			do_resize
										lowercase_  							=			size
										lowercase_  							=			resample
										lowercase_  							=			do_center_crop
										lowercase_  							=			crop_size
										lowercase_  							=			do_rescale
										lowercase_  							=			rescale_factor
										lowercase_  							=			do_normalize
										lowercase_  							=			image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
										lowercase_  							=			image_std if image_std is not None else IMAGENET_DEFAULT_STD

					def   UpperCAmelCase__       (     self       :   int							,					UpperCamelCase__       :   np.ndarray							,					UpperCamelCase__       :   Dict[str, int]							,					UpperCamelCase__       :   PILImageResampling = PILImageResampling.BICUBIC							,					UpperCamelCase__       :   Optional[Union[str, ChannelDimension]] = None							,					**UpperCamelCase__       :   Optional[int]							,					):






										'''simple docstring'''






										lowercase_  							=			get_size_dict(UpperCamelCase__							,					default_to_square=UpperCamelCase__       )
										# size_dict is a dict with either keys "height" and "width" or "shortest_edge"
										if "shortest_edge" in size:
															lowercase_  							=			int((256 / 224) * size["""shortest_edge"""]       )
															lowercase_  							=			get_resize_output_image_size(UpperCamelCase__							,					size=UpperCamelCase__							,					default_to_square=UpperCamelCase__       )
															lowercase_  							=			{"""height""": output_size[0], """width""": output_size[1]}
										if "height" not in size_dict or "width" not in size_dict:
															raise ValueError(
															    F'''Size dict must have keys \'height\' and \'width\' or \'shortest_edge\'. Got {size_dict.keys()}'''       )
										return resize(
										    UpperCamelCase__							,					size=(size_dict["""height"""], size_dict["""width"""])							,					resample=UpperCamelCase__							,					data_format=UpperCamelCase__							,					**UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   str							,					UpperCamelCase__       :   np.ndarray							,					UpperCamelCase__       :   Dict[str, int]							,					UpperCamelCase__       :   Optional[Union[str, ChannelDimension]] = None							,					**UpperCamelCase__       :   str							,					):






										'''simple docstring'''






										lowercase_  							=			get_size_dict(UpperCamelCase__       )
										if "height" not in size or "width" not in size:
															raise ValueError(F'''Size dict must have keys \'height\' and \'width\'. Got {size.keys()}'''       )
										return center_crop(UpperCamelCase__							,					size=(size["""height"""], size["""width"""])							,					data_format=UpperCamelCase__							,					**UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   Optional[int]							,					UpperCamelCase__       :   np.ndarray							,					UpperCamelCase__       :   Union[int, float]							,					UpperCamelCase__       :   Optional[Union[str, ChannelDimension]] = None							,					**UpperCamelCase__       :   str							,					):






										'''simple docstring'''






										return rescale(UpperCamelCase__							,					scale=UpperCamelCase__							,					data_format=UpperCamelCase__							,					**UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   List[str]							,					UpperCamelCase__       :   np.ndarray							,					UpperCamelCase__       :   Union[float, List[float]]							,					UpperCamelCase__       :   Union[float, List[float]]							,					UpperCamelCase__       :   Optional[Union[str, ChannelDimension]] = None							,					**UpperCamelCase__       :   Optional[int]							,					):






										'''simple docstring'''






										return normalize(UpperCamelCase__							,					mean=UpperCamelCase__							,					std=UpperCamelCase__							,					data_format=UpperCamelCase__							,					**UpperCamelCase__       )







					def   UpperCAmelCase__       (     self       :   Dict							,					UpperCamelCase__       :   ImageInput							,					UpperCamelCase__       :   Optional[bool] = None							,					UpperCamelCase__       :   Optional[Dict[str, int]] = None							,					UpperCamelCase__       :   PILImageResampling = None							,					UpperCamelCase__       :   Optional[bool] = None							,					UpperCamelCase__       :   Optional[Dict[str, int]] = None							,					UpperCamelCase__       :   Optional[bool] = None							,					UpperCamelCase__       :   Optional[float] = None							,					UpperCamelCase__       :   Optional[bool] = None							,					UpperCamelCase__       :   Optional[Union[float, Iterable[float]]] = None							,					UpperCamelCase__       :   Optional[Union[float, Iterable[float]]] = None							,					UpperCamelCase__       :   Optional[TensorType] = None							,					UpperCamelCase__       :   ChannelDimension = ChannelDimension.FIRST							,					**UpperCamelCase__       :   Optional[int]							,					):






										'''simple docstring'''






										lowercase_  							=			do_resize if do_resize is not None else self.do_resize
										lowercase_  							=			resample if resample is not None else self.resample
										lowercase_  							=			do_center_crop if do_center_crop is not None else self.do_center_crop
										lowercase_  							=			do_rescale if do_rescale is not None else self.do_rescale
										lowercase_  							=			rescale_factor if rescale_factor is not None else self.rescale_factor
										lowercase_  							=			do_normalize if do_normalize is not None else self.do_normalize
										lowercase_  							=			image_mean if image_mean is not None else self.image_mean
										lowercase_  							=			image_std if image_std is not None else self.image_std

										lowercase_  							=			size if size is not None else self.size
										lowercase_  							=			get_size_dict(UpperCamelCase__							,					default_to_square=UpperCamelCase__       )
										lowercase_  							=			crop_size if crop_size is not None else self.crop_size
										lowercase_  							=			get_size_dict(UpperCamelCase__							,					param_name="""crop_size"""       )

										lowercase_  							=			make_list_of_images(UpperCamelCase__       )

										if not valid_images(UpperCamelCase__       ):
															raise ValueError(
															    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
															    """torch.Tensor, tf.Tensor or jax.ndarray."""       )

										if do_resize and size is None:
															raise ValueError("""Size must be specified if do_resize is True."""       )

										if do_center_crop and crop_size is None:
															raise ValueError("""Crop size must be specified if do_center_crop is True."""       )

										if do_rescale and rescale_factor is None:
															raise ValueError("""Rescale factor must be specified if do_rescale is True."""       )

										if do_normalize and (image_mean is None or image_std is None):
															raise ValueError("""Image mean and std must be specified if do_normalize is True."""       )

										# All transformations expect numpy arrays.
										lowercase_  							=			[to_numpy_array(UpperCamelCase__       ) for image in images]

										if do_resize:
															lowercase_  							=			[self.resize(UpperCamelCase__							,					UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										if do_center_crop:
															lowercase_  							=			[self.center_crop(UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										if do_rescale:
															lowercase_  							=			[self.rescale(UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										if do_normalize:
															lowercase_  							=			[self.normalize(UpperCamelCase__							,					UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										lowercase_  							=			[to_channel_dimension_format(UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										lowercase_  							=			{"""pixel_values""": images}
										return BatchFeature(data=UpperCamelCase__							,					tensor_type=UpperCamelCase__       )


