


import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
				import torch


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


_UpperCAmelCase			:  Optional[Any]     =logging.get_logger(__name__)
_UpperCAmelCase			:  List[Any]     ={"""vocab_file""": """spiece.model"""}

_UpperCAmelCase			:  Union[str, Any]     ={
    """vocab_file""": {
        """AI-Sweden/gpt-sw3-126m""": """https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-350m""": """https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-1.6b""": """https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-6.7b""": """https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-20b""": """https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model""",
    }
}

_UpperCAmelCase			:  Optional[int]     ={
    """AI-Sweden/gpt-sw3-126m""": 2048,
    """AI-Sweden/gpt-sw3-350m""": 2048,
    """AI-Sweden/gpt-sw3-1.6b""": 2048,
    """AI-Sweden/gpt-sw3-6.7b""": 2048,
    """AI-Sweden/gpt-sw3-20b""": 2048,
}







class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  Dict					=							VOCAB_FILES_NAMES
						SCREAMING_SNAKE_CASE__      :  Optional[Any]					=							PRETRAINED_VOCAB_FILES_MAP
						SCREAMING_SNAKE_CASE__      :  List[str]					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						SCREAMING_SNAKE_CASE__      :  str					=							["""input_ids""", """attention_mask"""]




						def __init__(				self   ,						__lowercase   ,						__lowercase=False   ,						__lowercase=False   ,						__lowercase=False   ,						__lowercase=None   ,						__lowercase=None   ,						__lowercase=None   ,						__lowercase=None   ,						__lowercase = None   ,						**__lowercase   ,						)					->    None:
										lowerCAmelCase_ :      Optional[Any]    				=				{} if sp_model_kwargs is None else sp_model_kwargs

										lowerCAmelCase_ :      Union[str, Any]    				=				kwargs.get('''name_or_path'''							)
										if name_or_path is None:
														logger.warning(
														    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
														    ''' you are testing the model, this can safely be ignored'''							)
														lowerCAmelCase_ :      Optional[int]    				=				'''None'''

										# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
										lowerCAmelCase_ :      Union[str, Any]    				=				'''<|endoftext|>''' if eos_token is None else eos_token
										lowerCAmelCase_ :      str    				=				'''<unk>''' if unk_token is None else unk_token
										if "gpt-sw3-7b" in name_or_path:
														lowerCAmelCase_ :      Tuple    				=				unk_token if pad_token is None else pad_token
														lowerCAmelCase_ :      str    				=				eos_token if bos_token is None else bos_token
										else:
														lowerCAmelCase_ :      List[str]    				=				'''<pad>''' if pad_token is None else pad_token
														lowerCAmelCase_ :      List[str]    				=				'''<s>''' if bos_token is None else bos_token

										super().__init__(
										    do_lower_case=__lowercase   ,						remove_space=__lowercase   ,						keep_accents=__lowercase   ,						bos_token=__lowercase   ,						eos_token=__lowercase   ,						unk_token=__lowercase   ,						pad_token=__lowercase   ,						sp_model_kwargs=self.sp_model_kwargs   ,						**__lowercase   ,						)

										lowerCAmelCase_ :      Tuple    				=				do_lower_case
										lowerCAmelCase_ :      Optional[Any]    				=				remove_space
										lowerCAmelCase_ :      Tuple    				=				keep_accents
										lowerCAmelCase_ :      Optional[int]    				=				vocab_file

										lowerCAmelCase_ :      Optional[int]    				=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
										self.sp_model.Load(__lowercase							)

										# Used for whitespace normalization in input texts
										# fmt : off
										lowerCAmelCase_ :      Dict    				=				{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
										# fmt : on

										# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
										lowerCAmelCase_ :      Any    				=				re.compile(
										    f"""[{"".join(map(__lowercase   ,						list(range(0   ,						9							)							) + list(range(1_1   ,						3_2							)							) + list(range(1_2_7   ,						1_6_0							)							) + [1_6_0, 1_7_3, 8_2_0_3]							)							)}]"""							)




						def __getstate__(				self							)					->    List[Any]:
										lowerCAmelCase_ :      Optional[int]    				=				self.__dict__.copy()
										lowerCAmelCase_ :      Union[str, Any]    				=				None
										return state




						def __setstate__(				self   ,						__lowercase							)					->    List[Any]:
										lowerCAmelCase_ :      Union[str, Any]    				=				d

										# for backward compatibility
										if not hasattr(self   ,						'''sp_model_kwargs'''							):
														lowerCAmelCase_ :      List[str]    				=				{}

										lowerCAmelCase_ :      List[Any]    				=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
										self.sp_model.Load(self.vocab_file							)




						@property
						# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
						def 						lowercase_ (				self							)					->    int:
										return len(self.sp_model							)




						def 						lowercase_ (				self   ,						__lowercase							)					->    str:
										lowerCAmelCase_ :      List[Any]    				=				self.non_printing_characters_re.sub(''''''   ,						__lowercase							)

										# Normalize whitespaces
										lowerCAmelCase_ :      str    				=				''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]							)

										# NFC Unicode normalization
										lowerCAmelCase_ :      Any    				=				unicodedata.normalize('''NFC'''   ,						__lowercase							)
										return text




						def 						lowercase_ (				self   ,						__lowercase   ,						**__lowercase							)					->    List[str]:
										lowerCAmelCase_ :      Optional[int]    				=				self.preprocess_text(__lowercase							)
										return self.sp_model.encode(__lowercase   ,						out_type=__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase							)					->    int:
										return self.sp_model.PieceToId(__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase							)					->    str:
										return self.sp_model.IdToPiece(__lowercase							)




						@staticmethod
						def 						lowercase_ (				__lowercase							)					->    str:
										return out_string




						def 						lowercase_ (				self   ,						__lowercase							)					->    str:
										lowerCAmelCase_ :      str    				=				[]
										lowerCAmelCase_ :      Union[str, Any]    				=				''''''
										lowerCAmelCase_ :      str    				=				False
										for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																		# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																		if not prev_is_special:
																						out_string += " "

																		out_string += self.sp_model.decode(__lowercase							) + token
																		lowerCAmelCase_ :      Optional[Any]    				=				True
																		lowerCAmelCase_ :      Dict    				=				[]
														else:
																		current_sub_tokens.append(__lowercase							)
																		lowerCAmelCase_ :      str    				=				False
										out_string += self.sp_model.decode(__lowercase							)

										return out_string




						def 						lowercase_ (				self							)					->    Dict[str, int]:
										lowerCAmelCase_ :      List[Any]    				=				{self.convert_ids_to_tokens(__lowercase							): i for i in range(self.vocab_size							)}
										vocab.update(self.added_tokens_encoder							)
										return vocab




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None							)					->    Tuple[str]:
										if not os.path.isdir(__lowercase							):
														logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
														return
										lowerCAmelCase_ :      int    				=				os.path.join(
										    __lowercase   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

										if os.path.abspath(self.vocab_file							) != os.path.abspath(__lowercase							) and os.path.isfile(self.vocab_file							):
														copyfile(self.vocab_file   ,						__lowercase							)
										elif not os.path.isfile(self.vocab_file							):
														with open(__lowercase   ,						'''wb'''							) as fi:
																		lowerCAmelCase_ :      str    				=				self.sp_model.serialized_model_proto()
																		fi.write(__lowercase							)

										return (out_vocab_file,)




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = False							)					->    Union[List[int], List[List[int]], "torch.Tensor"]:

										if isinstance(__lowercase   ,						__lowercase							):
														lowerCAmelCase_ :      Optional[Any]    				=				self.preprocess_text(__lowercase							)
														lowerCAmelCase_ :      Optional[Any]    				=				self.sp_model.encode(__lowercase							)
										else:
														lowerCAmelCase_ :      Any    				=				[self.preprocess_text(__lowercase							) for t in text]
														lowerCAmelCase_ :      List[str]    				=				self.sp_model.encode(__lowercase							)

										if return_tensors is True or return_tensors == "pt":
														lowerCAmelCase_ :      int    				=				torch.tensor(__lowercase							)

										return token_ids




						def 						lowercase_ (				self   ,						__lowercase							)					->    str:
										return self.sp_model.decode(__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase							)					->    List[int]:
										lowerCAmelCase_ :      Optional[int]    				=				[f"""User: {text}""" if is_user else f"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
										lowerCAmelCase_ :      List[str]    				=				(
										    f"""{self.eos_token}{self.bos_token}""" + f"""{self.bos_token}""".join(__lowercase							) + f"""{self.bos_token}Bot:"""
										)
										return self.encode(text=__lowercase							)



from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
						import PIL


_UpperCAmelCase			:  Any     =logging.get_logger(__name__)







class 				snake_case__(							UpperCAmelCase__							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  Dict					=							["""pixel_values"""]




						def __init__(				self   ,						__lowercase = True   ,						__lowercase = None   ,						__lowercase = None   ,						__lowercase = PILImageResampling.BILINEAR   ,						__lowercase = True   ,						__lowercase = 1 / 2_5_5   ,						__lowercase = True   ,						__lowercase = None   ,						__lowercase = None   ,						**__lowercase   ,						)					->    None:
										super().__init__(**__lowercase							)
										lowerCAmelCase_ :      Dict    				=				size if size is not None else {'''shortest_edge''': 3_8_4}
										lowerCAmelCase_ :      Optional[Any]    				=				get_size_dict(__lowercase   ,						default_to_square=__lowercase							)

										lowerCAmelCase_ :      List[Any]    				=				do_resize
										lowerCAmelCase_ :      Optional[int]    				=				size
										# Default value set here for backwards compatibility where the value in config is None
										lowerCAmelCase_ :      str    				=				crop_pct if crop_pct is not None else 2_2_4 / 2_5_6
										lowerCAmelCase_ :      Tuple    				=				resample
										lowerCAmelCase_ :      Optional[int]    				=				do_rescale
										lowerCAmelCase_ :      Any    				=				rescale_factor
										lowerCAmelCase_ :      List[str]    				=				do_normalize
										lowerCAmelCase_ :      str    				=				image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
										lowerCAmelCase_ :      Dict    				=				image_std if image_std is not None else IMAGENET_STANDARD_STD




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase   ,						__lowercase   ,						__lowercase = PILImageResampling.BICUBIC   ,						__lowercase = None   ,						**__lowercase   ,						)					->    np.ndarray:
										lowerCAmelCase_ :      Optional[Any]    				=				get_size_dict(__lowercase   ,						default_to_square=__lowercase							)
										if "shortest_edge" not in size:
														raise ValueError(f"""Size dictionary must contain 'shortest_edge' key. Got {size.keys()}"""							)
										lowerCAmelCase_ :      Optional[int]    				=				size['''shortest_edge''']

										if shortest_edge < 3_8_4:
														# maintain same ratio, resizing shortest edge to shortest_edge/crop_pct
														lowerCAmelCase_ :      Optional[Any]    				=				int(shortest_edge / crop_pct							)
														lowerCAmelCase_ :      List[str]    				=				get_resize_output_image_size(__lowercase   ,						size=__lowercase   ,						default_to_square=__lowercase							)
														lowerCAmelCase_ :      List[Any]    				=				resize(image=__lowercase   ,						size=__lowercase   ,						resample=__lowercase   ,						data_format=__lowercase   ,						**__lowercase							)
														# then crop to (shortest_edge, shortest_edge)
														return center_crop(image=__lowercase   ,						size=(shortest_edge, shortest_edge)   ,						data_format=__lowercase   ,						**__lowercase							)
										else:
														# warping (no cropping) when evaluated at 384 or larger
														return resize(
														    __lowercase   ,						size=(shortest_edge, shortest_edge)   ,						resample=__lowercase   ,						data_format=__lowercase   ,						**__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase   ,						__lowercase = None   ,						**__lowercase   ,						)					->    Any:
										return rescale(__lowercase   ,						scale=__lowercase   ,						data_format=__lowercase   ,						**__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase   ,						__lowercase   ,						__lowercase = None   ,						**__lowercase   ,						)					->    np.ndarray:
										return normalize(__lowercase   ,						mean=__lowercase   ,						std=__lowercase   ,						data_format=__lowercase   ,						**__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase = None   ,						__lowercase = None   ,						__lowercase = None   ,						__lowercase = None   ,						__lowercase = None   ,						__lowercase = None   ,						__lowercase = None   ,						__lowercase = None   ,						__lowercase = None   ,						__lowercase = None   ,						__lowercase = ChannelDimension.FIRST   ,						**__lowercase   ,						)					->    PIL.Image.Image:
										lowerCAmelCase_ :      Optional[int]    				=				do_resize if do_resize is not None else self.do_resize
										lowerCAmelCase_ :      Any    				=				crop_pct if crop_pct is not None else self.crop_pct
										lowerCAmelCase_ :      str    				=				resample if resample is not None else self.resample
										lowerCAmelCase_ :      Union[str, Any]    				=				do_rescale if do_rescale is not None else self.do_rescale
										lowerCAmelCase_ :      str    				=				rescale_factor if rescale_factor is not None else self.rescale_factor
										lowerCAmelCase_ :      Any    				=				do_normalize if do_normalize is not None else self.do_normalize
										lowerCAmelCase_ :      str    				=				image_mean if image_mean is not None else self.image_mean
										lowerCAmelCase_ :      int    				=				image_std if image_std is not None else self.image_std

										lowerCAmelCase_ :      int    				=				size if size is not None else self.size
										lowerCAmelCase_ :      List[str]    				=				get_size_dict(__lowercase   ,						default_to_square=__lowercase							)

										lowerCAmelCase_ :      Tuple    				=				make_list_of_images(__lowercase							)

										if not valid_images(__lowercase							):
														raise ValueError(
														    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
														    '''torch.Tensor, tf.Tensor or jax.ndarray.'''							)

										if do_resize and size is None or resample is None:
														raise ValueError('''Size and resample must be specified if do_resize is True.'''							)

										if do_resize and size["shortest_edge"] < 3_8_4 and crop_pct is None:
														raise ValueError('''crop_pct must be specified if size < 384.'''							)

										if do_rescale and rescale_factor is None:
														raise ValueError('''Rescale factor must be specified if do_rescale is True.'''							)

										if do_normalize and (image_mean is None or image_std is None):
														raise ValueError('''Image mean and std must be specified if do_normalize is True.'''							)

										# All transformations expect numpy arrays.
										lowerCAmelCase_ :      Optional[Any]    				=				[to_numpy_array(__lowercase							) for image in images]

										if do_resize:
														lowerCAmelCase_ :      Union[str, Any]    				=				[self.resize(image=__lowercase   ,						size=__lowercase   ,						crop_pct=__lowercase   ,						resample=__lowercase							) for image in images]

										if do_rescale:
														lowerCAmelCase_ :      Any    				=				[self.rescale(image=__lowercase   ,						scale=__lowercase							) for image in images]

										if do_normalize:
														lowerCAmelCase_ :      List[Any]    				=				[self.normalize(image=__lowercase   ,						mean=__lowercase   ,						std=__lowercase							) for image in images]

										lowerCAmelCase_ :      Optional[Any]    				=				[to_channel_dimension_format(__lowercase   ,						__lowercase							) for image in images]

										lowerCAmelCase_ :      Dict    				=				{'''pixel_values''': images}
										return BatchFeature(data=__lowercase   ,						tensor_type=__lowercase							)