





'''simple docstring'''







import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
					import torch


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


snake_case_     :       int						=  logging.get_logger(__name__)
snake_case_     :       Union[str, Any]						=  {"vocab_file": "spiece.model"}

snake_case_     :       Dict						=  {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

snake_case_     :       Optional[int]						=  {
    "AI-Sweden/gpt-sw3-126m": 20_48,
    "AI-Sweden/gpt-sw3-350m": 20_48,
    "AI-Sweden/gpt-sw3-1.6b": 20_48,
    "AI-Sweden/gpt-sw3-6.7b": 20_48,
    "AI-Sweden/gpt-sw3-20b": 20_48,
}


class 					__a      (lowerCamelCase     ):
							__a					:   List[str]									=       VOCAB_FILES_NAMES
							__a					:   int									=       PRETRAINED_VOCAB_FILES_MAP
							__a					:   Optional[int]									=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a					:   Optional[int]									=       ["input_ids", "attention_mask"]

							def __init__(							self							:     Optional[int]       ,     __magic_name__							:     List[str]       ,     __magic_name__							:     str=False       ,     __magic_name__							:     List[str]=False       ,     __magic_name__							:     Any=False       ,     __magic_name__							:     Tuple=None       ,     __magic_name__							:     str=None       ,     __magic_name__							:     Union[str, Any]=None       ,     __magic_name__							:     int=None       ,     __magic_name__							:     Optional[Dict[str, Any]] = None       ,     **__magic_name__							:     List[str]       ,     )					->       None:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[Any]     						=  {} if sp_model_kwargs is None else sp_model_kwargs

													UpperCAmelCase_			:      Union[str, Any]     						=  kwargs.get('''name_or_path'''   )
													if name_or_path is None:
																			logger.warning(
																			    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
																			    ''' you are testing the model, this can safely be ignored'''   )
																			UpperCAmelCase_			:      Optional[Any]     						=  '''None'''

													# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
													UpperCAmelCase_			:      Any     						=  '''<|endoftext|>''' if eos_token is None else eos_token
													UpperCAmelCase_			:      Dict     						=  '''<unk>''' if unk_token is None else unk_token
													if "gpt-sw3-7b" in name_or_path:
																			UpperCAmelCase_			:      Optional[int]     						=  unk_token if pad_token is None else pad_token
																			UpperCAmelCase_			:      Any     						=  eos_token if bos_token is None else bos_token
													else:
																			UpperCAmelCase_			:      List[str]     						=  '''<pad>''' if pad_token is None else pad_token
																			UpperCAmelCase_			:      Optional[int]     						=  '''<s>''' if bos_token is None else bos_token

													super().__init__(
													    do_lower_case=__magic_name__       ,     remove_space=__magic_name__       ,     keep_accents=__magic_name__       ,     bos_token=__magic_name__       ,     eos_token=__magic_name__       ,     unk_token=__magic_name__       ,     pad_token=__magic_name__       ,     sp_model_kwargs=self.sp_model_kwargs       ,     **__magic_name__       ,     )

													UpperCAmelCase_			:      Any     						=  do_lower_case
													UpperCAmelCase_			:      Union[str, Any]     						=  remove_space
													UpperCAmelCase_			:      List[str]     						=  keep_accents
													UpperCAmelCase_			:      Tuple     						=  vocab_file

													UpperCAmelCase_			:      List[Any]     						=  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
													self.sp_model.Load(__magic_name__   )

													# Used for whitespace normalization in input texts
													# fmt : off
													UpperCAmelCase_			:      int     						=  {''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
													# fmt : on

													# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
													UpperCAmelCase_			:      Union[str, Any]     						=  re.compile(
													    F"""[{"".join(map(__magic_name__       ,     list(range(0       ,     9   )   ) + list(range(11       ,     32   )   ) + list(range(1_27       ,     1_60   )   ) + [1_60, 1_73, 82_03]   )   )}]"""   )

							def __getstate__(							self							:     List[Any]   )					->       Union[str, Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      int     						=  self.__dict__.copy()
													UpperCAmelCase_			:      str     						=  None
													return state

							def __setstate__(							self							:     Union[str, Any]       ,     __magic_name__							:     Tuple   )					->       int:







													"""simple docstring"""
													UpperCAmelCase_			:      Union[str, Any]     						=  d

													# for backward compatibility
													if not hasattr(self       ,     '''sp_model_kwargs'''   ):
																			UpperCAmelCase_			:      Union[str, Any]     						=  {}

													UpperCAmelCase_			:      Optional[int]     						=  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
													self.sp_model.Load(self.vocab_file   )

							@property
							# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
							def     UpperCAmelCase__	(							self							:     str   )					->       int:







													"""simple docstring"""
													return len(self.sp_model   )

							def     UpperCAmelCase__	(							self							:     Optional[Any]       ,     __magic_name__							:     str   )					->       str:







													"""simple docstring"""
													UpperCAmelCase_			:      List[str]     						=  self.non_printing_characters_re.sub(''''''       ,     __magic_name__   )

													# Normalize whitespaces
													UpperCAmelCase_			:      Any     						=  ''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]   )

													# NFC Unicode normalization
													UpperCAmelCase_			:      Union[str, Any]     						=  unicodedata.normalize('''NFC'''       ,     __magic_name__   )
													return text

							def     UpperCAmelCase__	(							self							:     Tuple       ,     __magic_name__							:     str       ,     **__magic_name__							:     List[Any]   )					->       List[str]:







													"""simple docstring"""
													UpperCAmelCase_			:      List[Any]     						=  self.preprocess_text(__magic_name__   )
													return self.sp_model.encode(__magic_name__       ,     out_type=__magic_name__   )

							def     UpperCAmelCase__	(							self							:     Optional[Any]       ,     __magic_name__							:     str   )					->       int:







													"""simple docstring"""
													return self.sp_model.PieceToId(__magic_name__   )

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     int   )					->       str:







													"""simple docstring"""
													return self.sp_model.IdToPiece(__magic_name__   )

							@staticmethod
							def     UpperCAmelCase__	(							__magic_name__							:     str   )					->       str:







													"""simple docstring"""
													return out_string

							def     UpperCAmelCase__	(							self							:     Tuple       ,     __magic_name__							:     List[str]   )					->       str:







													"""simple docstring"""
													UpperCAmelCase_			:      Dict     						=  []
													UpperCAmelCase_			:      Any     						=  ''''''
													UpperCAmelCase_			:      int     						=  False
													for token in tokens:
																			# make sure that special tokens are not decoded using sentencepiece model
																			if token in self.all_special_tokens:
																									# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																									if not prev_is_special:
																															out_string += " "

																									out_string += self.sp_model.decode(__magic_name__   ) + token
																									UpperCAmelCase_			:      Optional[Any]     						=  True
																									UpperCAmelCase_			:      int     						=  []
																			else:
																									current_sub_tokens.append(__magic_name__   )
																									UpperCAmelCase_			:      str     						=  False
													out_string += self.sp_model.decode(__magic_name__   )

													return out_string

							def     UpperCAmelCase__	(							self							:     Optional[Any]   )					->       Dict[str, int]:







													"""simple docstring"""
													UpperCAmelCase_			:      List[str]     						=  {self.convert_ids_to_tokens(__magic_name__   ): i for i in range(self.vocab_size   )}
													vocab.update(self.added_tokens_encoder   )
													return vocab

							def     UpperCAmelCase__	(							self							:     Optional[int]       ,     __magic_name__							:     str       ,     __magic_name__							:     Optional[str] = None   )					->       Tuple[str]:







													"""simple docstring"""
													if not os.path.isdir(__magic_name__   ):
																			logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
																			return
													UpperCAmelCase_			:      Union[str, Any]     						=  os.path.join(
													    __magic_name__       ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

													if os.path.abspath(self.vocab_file   ) != os.path.abspath(__magic_name__   ) and os.path.isfile(self.vocab_file   ):
																			copyfile(self.vocab_file       ,     __magic_name__   )
													elif not os.path.isfile(self.vocab_file   ):
																			with open(__magic_name__       ,     '''wb'''   ) as fi:
																									UpperCAmelCase_			:      List[Any]     						=  self.sp_model.serialized_model_proto()
																									fi.write(__magic_name__   )

													return (out_vocab_file,)

							def     UpperCAmelCase__	(							self							:     List[str]       ,     __magic_name__							:     Union[str, List[str]]       ,     __magic_name__							:     Union[str, bool] = False   )					->       Union[List[int], List[List[int]], "torch.Tensor"]:







													"""simple docstring"""

													if isinstance(__magic_name__       ,     __magic_name__   ):
																			UpperCAmelCase_			:      Any     						=  self.preprocess_text(__magic_name__   )
																			UpperCAmelCase_			:      str     						=  self.sp_model.encode(__magic_name__   )
													else:
																			UpperCAmelCase_			:      Any     						=  [self.preprocess_text(__magic_name__   ) for t in text]
																			UpperCAmelCase_			:      Optional[Any]     						=  self.sp_model.encode(__magic_name__   )

													if return_tensors is True or return_tensors == "pt":
																			UpperCAmelCase_			:      Tuple     						=  torch.tensor(__magic_name__   )

													return token_ids

							def     UpperCAmelCase__	(							self							:     Dict       ,     __magic_name__							:     Union[int, List[int]]   )					->       str:







													"""simple docstring"""
													return self.sp_model.decode(__magic_name__   )



							def     UpperCAmelCase__	(							self							:     Union[str, Any]       ,     __magic_name__							:     "Conversation"   )					->       List[int]:







													"""simple docstring"""
													UpperCAmelCase_			:      Optional[int]     						=  [F"""User: {text}""" if is_user else F"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
													UpperCAmelCase_			:      Any     						=  (
													    F"""{self.eos_token}{self.bos_token}""" + F"""{self.bos_token}""".join(__magic_name__   ) + F"""{self.bos_token}Bot:"""
													)
													return self.encode(text=__magic_name__   )












'''simple docstring'''







from typing import List, Optional, Union

import numpy as np
import PIL.Image

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


snake_case_     :       Union[str, Any]						=  logging.get_logger(__name__)


class 					__a      (lowerCamelCase     ):
							__a					:   Tuple									=       ["pixel_values"]

							def __init__(							self							:     List[Any]       ,     __magic_name__							:     bool = True       ,     __magic_name__							:     int = 32       ,     __magic_name__							:     Union[str, Any]=PILImageResampling.BILINEAR       ,     __magic_name__							:     bool = True       ,     **__magic_name__							:     List[str]       ,     )					->       None:







													"""simple docstring"""
													UpperCAmelCase_			:      int     						=  do_resize
													UpperCAmelCase_			:      Tuple     						=  do_rescale
													UpperCAmelCase_			:      List[Any]     						=  size_divisor
													UpperCAmelCase_			:      Any     						=  resample
													super().__init__(**__magic_name__   )

							def     UpperCAmelCase__	(							self							:     Optional[Any]       ,     __magic_name__							:     np.ndarray       ,     __magic_name__							:     int       ,     __magic_name__							:     str       ,     __magic_name__							:     Optional[ChannelDimension] = None       ,     **__magic_name__							:     Tuple   )					->       np.ndarray:







													"""simple docstring"""
													UpperCAmelCase_						,       UpperCAmelCase_			:      List[str]     						=  get_image_size(__magic_name__   )
													# Rounds the height and width down to the closest multiple of size_divisor
													UpperCAmelCase_			:      Dict     						=  height // size_divisor * size_divisor
													UpperCAmelCase_			:      Dict     						=  width // size_divisor * size_divisor
													UpperCAmelCase_			:      Any     						=  resize(__magic_name__       ,     (new_h, new_w)       ,     resample=__magic_name__       ,     data_format=__magic_name__       ,     **__magic_name__   )
													return image

							def     UpperCAmelCase__	(							self							:     int       ,     __magic_name__							:     np.ndarray       ,     __magic_name__							:     float       ,     __magic_name__							:     Optional[ChannelDimension] = None       ,     **__magic_name__							:     Optional[Any]   )					->       np.ndarray:







													"""simple docstring"""
													return rescale(image=__magic_name__       ,     scale=__magic_name__       ,     data_format=__magic_name__       ,     **__magic_name__   )



							def     UpperCAmelCase__	(							self							:     str       ,     __magic_name__							:     Union["PIL.Image.Image", TensorType, List["PIL.Image.Image"], List[TensorType]]       ,     __magic_name__							:     Optional[bool] = None       ,     __magic_name__							:     Optional[int] = None       ,     __magic_name__							:     Any=None       ,     __magic_name__							:     Optional[bool] = None       ,     __magic_name__							:     Optional[Union[TensorType, str]] = None       ,     __magic_name__							:     ChannelDimension = ChannelDimension.FIRST       ,     **__magic_name__							:     Tuple       ,     )					->       BatchFeature:







													"""simple docstring"""
													UpperCAmelCase_			:      Dict     						=  do_resize if do_resize is not None else self.do_resize
													UpperCAmelCase_			:      str     						=  do_rescale if do_rescale is not None else self.do_rescale
													UpperCAmelCase_			:      Any     						=  size_divisor if size_divisor is not None else self.size_divisor
													UpperCAmelCase_			:      Dict     						=  resample if resample is not None else self.resample

													if do_resize and size_divisor is None:
																			raise ValueError('''size_divisor is required for resizing'''   )

													UpperCAmelCase_			:      Optional[int]     						=  make_list_of_images(__magic_name__   )

													if not valid_images(__magic_name__   ):
																			raise ValueError('''Invalid image(s)'''   )

													# All transformations expect numpy arrays.
													UpperCAmelCase_			:      List[str]     						=  [to_numpy_array(__magic_name__   ) for img in images]

													if do_resize:
																			UpperCAmelCase_			:      str     						=  [self.resize(__magic_name__       ,     size_divisor=__magic_name__       ,     resample=__magic_name__   ) for image in images]

													if do_rescale:
																			UpperCAmelCase_			:      Tuple     						=  [self.rescale(__magic_name__       ,     scale=1 / 2_55   ) for image in images]

													UpperCAmelCase_			:      Union[str, Any]     						=  [to_channel_dimension_format(__magic_name__       ,     __magic_name__   ) for image in images]

													UpperCAmelCase_			:      int     						=  {'''pixel_values''': images}
													return BatchFeature(data=__magic_name__       ,     tensor_type=__magic_name__   )





