


'''simple docstring'''







from typing import Optional

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import (
    BackboneOutput,
    BaseModelOutputWithNoAttention,
    BaseModelOutputWithPoolingAndNoAttention,
    ImageClassifierOutputWithNoAttention,
)
from ...modeling_utils import PreTrainedModel
from ...utils import (
    add_code_sample_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    logging,
    replace_return_docstrings,
)
from ...utils.backbone_utils import BackboneMixin
from .configuration_resnet import ResNetConfig


lowerCAmelCase    :     List[str]					=						logging.get_logger(__name__)

# General docstring
lowerCAmelCase    :     List[str]					=						"""ResNetConfig"""

# Base docstring
lowerCAmelCase    :     List[Any]					=						"""microsoft/resnet-50"""
lowerCAmelCase    :     List[Any]					=						[1, 20_48, 7, 7]

# Image classification docstring
lowerCAmelCase    :     Any					=						"""microsoft/resnet-50"""
lowerCAmelCase    :     str					=						"""tiger cat"""

lowerCAmelCase    :     List[Any]					=						[
    """microsoft/resnet-50""",
    # See all resnet models at https://huggingface.co/models?filter=resnet
]
class 	UpperCamelCase__					( nn.Module	):



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__      ,	snake_case__ = 3      ,	snake_case__ = 1      ,	snake_case__ = "relu"   ):

            '''simple docstring'''

            super().__init__()
            _lowerCAmelCase    :							Dict   			=       nn.Convad(
                snake_case__      ,	snake_case__      ,	kernel_size=snake_case__      ,	stride=snake_case__      ,	padding=kernel_size // 2      ,	bias=snake_case__   )
            _lowerCAmelCase    :							Any   			=       nn.BatchNormad(snake_case__   )
            _lowerCAmelCase    :							str   			=       ACTaFN[activation] if activation is not None else nn.Identity()







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       self.convolution(snake_case__   )
            _lowerCAmelCase    :							Union[str, Any]   			=       self.normalization(snake_case__   )
            _lowerCAmelCase    :							List[str]   			=       self.activation(snake_case__   )
            return hidden_state
class 	UpperCamelCase__					( nn.Module	):



     """simple docstring"""






     def __init__( self      ,	snake_case__   ):

            '''simple docstring'''

            super().__init__()
            _lowerCAmelCase    :							List[str]   			=       ResNetConvLayer(
                config.num_channels      ,	config.embedding_size      ,	kernel_size=7      ,	stride=2      ,	activation=config.hidden_act   )
            _lowerCAmelCase    :							int   			=       nn.MaxPoolad(kernel_size=3      ,	stride=2      ,	padding=1   )
            _lowerCAmelCase    :							Tuple   			=       config.num_channels







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Tuple   			=       pixel_values.shape[1]
            if num_channels != self.num_channels:
                   raise ValueError(
                       'Make sure that the channel dimension of the pixel values match with the one set in the configuration.'   )
            _lowerCAmelCase    :							Dict   			=       self.embedder(snake_case__   )
            _lowerCAmelCase    :							Union[str, Any]   			=       self.pooler(snake_case__   )
            return embedding
class 	UpperCamelCase__					( nn.Module	):



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__      ,	snake_case__ = 2   ):

            '''simple docstring'''

            super().__init__()
            _lowerCAmelCase    :							List[str]   			=       nn.Convad(snake_case__      ,	snake_case__      ,	kernel_size=1      ,	stride=snake_case__      ,	bias=snake_case__   )
            _lowerCAmelCase    :							Tuple   			=       nn.BatchNormad(snake_case__   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       self.convolution(snake_case__   )
            _lowerCAmelCase    :							Optional[int]   			=       self.normalization(snake_case__   )
            return hidden_state
class 	UpperCamelCase__					( nn.Module	):



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__      ,	snake_case__ = 1      ,	snake_case__ = "relu"   ):

            '''simple docstring'''

            super().__init__()
            _lowerCAmelCase    :							int   			=       in_channels != out_channels or stride != 1
            _lowerCAmelCase    :							str   			=       (
                ResNetShortCut(snake_case__      ,	snake_case__      ,	stride=snake_case__   ) if should_apply_shortcut else nn.Identity()
            )
            _lowerCAmelCase    :							Optional[int]   			=       nn.Sequential(
                ResNetConvLayer(snake_case__      ,	snake_case__      ,	stride=snake_case__   )      ,	ResNetConvLayer(snake_case__      ,	snake_case__      ,	activation=snake_case__   )      ,	)
            _lowerCAmelCase    :							Optional[int]   			=       ACTaFN[activation]







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       hidden_state
            _lowerCAmelCase    :							List[Any]   			=       self.layer(snake_case__   )
            _lowerCAmelCase    :							int   			=       self.shortcut(snake_case__   )
            hidden_state += residual
            _lowerCAmelCase    :							List[Any]   			=       self.activation(snake_case__   )
            return hidden_state
class 	UpperCamelCase__					( nn.Module	):



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__      ,	snake_case__ = 1      ,	snake_case__ = "relu"      ,	snake_case__ = 4   ):

            '''simple docstring'''

            super().__init__()
            _lowerCAmelCase    :							Dict   			=       in_channels != out_channels or stride != 1
            _lowerCAmelCase    :							Any   			=       out_channels // reduction
            _lowerCAmelCase    :							List[str]   			=       (
                ResNetShortCut(snake_case__      ,	snake_case__      ,	stride=snake_case__   ) if should_apply_shortcut else nn.Identity()
            )
            _lowerCAmelCase    :							Optional[int]   			=       nn.Sequential(
                ResNetConvLayer(snake_case__      ,	snake_case__      ,	kernel_size=1   )      ,	ResNetConvLayer(snake_case__      ,	snake_case__      ,	stride=snake_case__   )      ,	ResNetConvLayer(snake_case__      ,	snake_case__      ,	kernel_size=1      ,	activation=snake_case__   )      ,	)
            _lowerCAmelCase    :							List[Any]   			=       ACTaFN[activation]







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       hidden_state
            _lowerCAmelCase    :							Optional[Any]   			=       self.layer(snake_case__   )
            _lowerCAmelCase    :							List[str]   			=       self.shortcut(snake_case__   )
            hidden_state += residual
            _lowerCAmelCase    :							str   			=       self.activation(snake_case__   )
            return hidden_state
class 	UpperCamelCase__					( nn.Module	):



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__ = 2      ,	snake_case__ = 2      ,	):

            '''simple docstring'''

            super().__init__()

            _lowerCAmelCase    :							Union[str, Any]   			=       ResNetBottleNeckLayer if config.layer_type == 'bottleneck' else ResNetBasicLayer

            _lowerCAmelCase    :							Tuple   			=       nn.Sequential(
                # downsampling is done in the first layer with stride of 2
                layer(snake_case__      ,	snake_case__      ,	stride=snake_case__      ,	activation=config.hidden_act   )      ,	*[layer(snake_case__      ,	snake_case__      ,	activation=config.hidden_act   ) for _ in range(depth - 1   )]      ,	)







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       input
            for layer in self.layers:
                   _lowerCAmelCase    :							Any   			=       layer(snake_case__   )
            return hidden_state
class 	UpperCamelCase__					( nn.Module	):



     """simple docstring"""






     def __init__( self      ,	snake_case__   ):

            '''simple docstring'''

            super().__init__()
            _lowerCAmelCase    :							Any   			=       nn.ModuleList([]   )
            # based on `downsample_in_first_stage` the first layer of the first stage may or may not downsample the input
            self.stages.append(
                ResNetStage(
                    snake_case__      ,	config.embedding_size      ,	config.hidden_sizes[0]      ,	stride=2 if config.downsample_in_first_stage else 1      ,	depth=config.depths[0]      ,	)   )
            _lowerCAmelCase    :							List[str]   			=       zip(config.hidden_sizes      ,	config.hidden_sizes[1:]   )
            for (in_channels, out_channels), depth in zip(snake_case__      ,	config.depths[1:]   ):
                   self.stages.append(ResNetStage(snake_case__      ,	snake_case__      ,	snake_case__      ,	depth=snake_case__   )   )







     def a	( self      ,	snake_case__      ,	snake_case__ = False      ,	snake_case__ = True   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       () if output_hidden_states else None

            for stage_module in self.stages:
                   if output_hidden_states:
                          _lowerCAmelCase    :							List[str]   			=       hidden_states + (hidden_state,)

                   _lowerCAmelCase    :							int   			=       stage_module(snake_case__   )

            if output_hidden_states:
                   _lowerCAmelCase    :							Optional[Any]   			=       hidden_states + (hidden_state,)

            if not return_dict:
                   return tuple(v for v in [hidden_state, hidden_states] if v is not None   )

            return BaseModelOutputWithNoAttention(
                last_hidden_state=snake_case__      ,	hidden_states=snake_case__      ,	)
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     __magic_name__    			=    ResNetConfig
     __magic_name__    			=    "resnet"
     __magic_name__    			=    "pixel_values"
     __magic_name__    			=    True







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            if isinstance(snake_case__      ,	nn.Convad   ):
                   nn.init.kaiming_normal_(module.weight      ,	mode='fan_out'      ,	nonlinearity='relu'   )
            elif isinstance(snake_case__      ,	(nn.BatchNormad, nn.GroupNorm)   ):
                   nn.init.constant_(module.weight      ,	1   )
                   nn.init.constant_(module.bias      ,	0   )







     def a	( self      ,	snake_case__      ,	snake_case__=False   ):

            '''simple docstring'''

            if isinstance(snake_case__      ,	snake_case__   ):
                   _lowerCAmelCase    :							Dict   			=       value
lowerCAmelCase    :     int					=						r"""
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`ResNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""

lowerCAmelCase    :     Optional[Any]					=						r"""
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`ConvNextImageProcessor.__call__`] for details.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
"""
@add_start_docstrings(
    "The bare ResNet model outputting raw features without any specific head on top."    ,					SCREAMING_SNAKE_CASE_    ,					)
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     def __init__( self      ,	snake_case__   ):

            '''simple docstring'''

            super().__init__(snake_case__   )
            _lowerCAmelCase    :							Optional[Any]   			=       config
            _lowerCAmelCase    :							str   			=       ResNetEmbeddings(snake_case__   )
            _lowerCAmelCase    :							Dict   			=       ResNetEncoder(snake_case__   )
            _lowerCAmelCase    :							Optional[int]   			=       nn.AdaptiveAvgPoolad((1, 1)   )
            # Initialize weights and apply final processing
            self.post_init()







     @add_start_docstrings_to_model_forward(snake_case__   )
     @add_code_sample_docstrings(
         checkpoint=_CHECKPOINT_FOR_DOC      ,	output_type=snake_case__      ,	config_class=_CONFIG_FOR_DOC      ,	modality='vision'      ,	expected_output=_EXPECTED_OUTPUT_SHAPE      ,	)
     def a	( self      ,	snake_case__      ,	snake_case__ = None      ,	snake_case__ = None   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       (
                output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
            )
            _lowerCAmelCase    :							int   			=       return_dict if return_dict is not None else self.config.use_return_dict

            _lowerCAmelCase    :							Optional[int]   			=       self.embedder(snake_case__   )

            _lowerCAmelCase    :							str   			=       self.encoder(
                snake_case__      ,	output_hidden_states=snake_case__      ,	return_dict=snake_case__   )

            _lowerCAmelCase    :							Dict   			=       encoder_outputs[0]

            _lowerCAmelCase    :							int   			=       self.pooler(snake_case__   )

            if not return_dict:
                   return (last_hidden_state, pooled_output) + encoder_outputs[1:]

            return BaseModelOutputWithPoolingAndNoAttention(
                last_hidden_state=snake_case__      ,	pooler_output=snake_case__      ,	hidden_states=encoder_outputs.hidden_states      ,	)
@add_start_docstrings(
    "\n    ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    "    ,					SCREAMING_SNAKE_CASE_    ,					)
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     def __init__( self      ,	snake_case__   ):

            '''simple docstring'''

            super().__init__(snake_case__   )
            _lowerCAmelCase    :							str   			=       config.num_labels
            _lowerCAmelCase    :							Union[str, Any]   			=       ResNetModel(snake_case__   )
            # classification head
            _lowerCAmelCase    :							Dict   			=       nn.Sequential(
                nn.Flatten()      ,	nn.Linear(config.hidden_sizes[-1]      ,	config.num_labels   ) if config.num_labels > 0 else nn.Identity()      ,	)
            # initialize weights and apply final processing
            self.post_init()







     @add_start_docstrings_to_model_forward(snake_case__   )
     @add_code_sample_docstrings(
         checkpoint=_IMAGE_CLASS_CHECKPOINT      ,	output_type=snake_case__      ,	config_class=_CONFIG_FOR_DOC      ,	expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT      ,	)
     def a	( self      ,	snake_case__ = None      ,	snake_case__ = None      ,	snake_case__ = None      ,	snake_case__ = None      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							Dict   			=       return_dict if return_dict is not None else self.config.use_return_dict

            _lowerCAmelCase    :							Any   			=       self.resnet(snake_case__      ,	output_hidden_states=snake_case__      ,	return_dict=snake_case__   )

            _lowerCAmelCase    :							List[Any]   			=       outputs.pooler_output if return_dict else outputs[1]

            _lowerCAmelCase    :							Optional[Any]   			=       self.classifier(snake_case__   )

            _lowerCAmelCase    :							Tuple   			=       None

            if labels is not None:
                   if self.config.problem_type is None:
                          if self.num_labels == 1:
                                 _lowerCAmelCase    :							int   			=       'regression'
                          elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
                                 _lowerCAmelCase    :							Any   			=       'single_label_classification'
                          else:
                                 _lowerCAmelCase    :							Tuple   			=       'multi_label_classification'
                   if self.config.problem_type == "regression":
                          _lowerCAmelCase    :							List[Any]   			=       MSELoss()
                          if self.num_labels == 1:
                                 _lowerCAmelCase    :							Tuple   			=       loss_fct(logits.squeeze()      ,	labels.squeeze()   )
                          else:
                                 _lowerCAmelCase    :							Optional[Any]   			=       loss_fct(snake_case__      ,	snake_case__   )
                   elif self.config.problem_type == "single_label_classification":
                          _lowerCAmelCase    :							str   			=       CrossEntropyLoss()
                          _lowerCAmelCase    :							Union[str, Any]   			=       loss_fct(logits.view(-1      ,	self.num_labels   )      ,	labels.view(-1   )   )
                   elif self.config.problem_type == "multi_label_classification":
                          _lowerCAmelCase    :							int   			=       BCEWithLogitsLoss()
                          _lowerCAmelCase    :							Dict   			=       loss_fct(snake_case__      ,	snake_case__   )

            if not return_dict:
                   _lowerCAmelCase    :							int   			=       (logits,) + outputs[2:]
                   return (loss,) + output if loss is not None else output

            return ImageClassifierOutputWithNoAttention(loss=snake_case__      ,	logits=snake_case__      ,	hidden_states=outputs.hidden_states   )
@add_start_docstrings(
    "\n    ResNet backbone, to be used with frameworks like DETR and MaskFormer.\n    "    ,					SCREAMING_SNAKE_CASE_    ,					)
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_    ,					SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     def __init__( self      ,	snake_case__   ):

            '''simple docstring'''

            super().__init__(snake_case__   )
            super()._init_backbone(snake_case__   )

            _lowerCAmelCase    :							List[str]   			=       [config.embedding_size] + config.hidden_sizes
            _lowerCAmelCase    :							Dict   			=       ResNetEmbeddings(snake_case__   )
            _lowerCAmelCase    :							str   			=       ResNetEncoder(snake_case__   )

            # initialize weights and apply final processing
            self.post_init()







     @add_start_docstrings_to_model_forward(snake_case__   )
     @replace_return_docstrings(output_type=snake_case__      ,	config_class=_CONFIG_FOR_DOC   )
     def a	( self      ,	snake_case__      ,	snake_case__ = None      ,	snake_case__ = None   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       return_dict if return_dict is not None else self.config.use_return_dict
            _lowerCAmelCase    :							Tuple   			=       (
                output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
            )

            _lowerCAmelCase    :							Optional[int]   			=       self.embedder(snake_case__   )

            _lowerCAmelCase    :							Any   			=       self.encoder(snake_case__      ,	output_hidden_states=snake_case__      ,	return_dict=snake_case__   )

            _lowerCAmelCase    :							List[Any]   			=       outputs.hidden_states

            _lowerCAmelCase    :							Optional[int]   			=       ()
            for idx, stage in enumerate(self.stage_names   ):
                   if stage in self.out_features:
                          feature_maps += (hidden_states[idx],)

            if not return_dict:
                   _lowerCAmelCase    :							Union[str, Any]   			=       (feature_maps,)
                   if output_hidden_states:
                          output += (outputs.hidden_states,)
                   return output

            return BackboneOutput(
                feature_maps=snake_case__      ,	hidden_states=outputs.hidden_states if output_hidden_states else None      ,	attentions=snake_case__      ,	)





'''simple docstring'''







import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
       import torch


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation


lowerCAmelCase    :     int					=						logging.get_logger(__name__)
lowerCAmelCase    :     Union[str, Any]					=						{"""vocab_file""": """spiece.model"""}

lowerCAmelCase    :     Optional[int]					=						{
    """vocab_file""": {
        """AI-Sweden/gpt-sw3-126m""": """https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-350m""": """https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-1.6b""": """https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-6.7b""": """https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-20b""": """https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model""",
    }
}

lowerCAmelCase    :     Union[str, Any]					=						{
    """AI-Sweden/gpt-sw3-126m""": 20_48,
    """AI-Sweden/gpt-sw3-350m""": 20_48,
    """AI-Sweden/gpt-sw3-1.6b""": 20_48,
    """AI-Sweden/gpt-sw3-6.7b""": 20_48,
    """AI-Sweden/gpt-sw3-20b""": 20_48,
}
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     __magic_name__    			=    VOCAB_FILES_NAMES
     __magic_name__    			=    PRETRAINED_VOCAB_FILES_MAP
     __magic_name__    			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__    			=    ["input_ids", "attention_mask"]







     def __init__( self      ,	snake_case__      ,	snake_case__=False      ,	snake_case__=False      ,	snake_case__=False      ,	snake_case__=None      ,	snake_case__=None      ,	snake_case__=None      ,	snake_case__=None      ,	snake_case__ = None      ,	**snake_case__      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							Tuple   			=       {} if sp_model_kwargs is None else sp_model_kwargs

            _lowerCAmelCase    :							List[Any]   			=       kwargs.get('name_or_path'   )
            if name_or_path is None:
                   logger.warning(
                       'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
                       ' you are testing the model, this can safely be ignored'   )
                   _lowerCAmelCase    :							Any   			=       'None'

            # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
            _lowerCAmelCase    :							str   			=       '<|endoftext|>' if eos_token is None else eos_token
            _lowerCAmelCase    :							Tuple   			=       '<unk>' if unk_token is None else unk_token
            if "gpt-sw3-7b" in name_or_path:
                   _lowerCAmelCase    :							List[str]   			=       unk_token if pad_token is None else pad_token
                   _lowerCAmelCase    :							Optional[int]   			=       eos_token if bos_token is None else bos_token
            else:
                   _lowerCAmelCase    :							Tuple   			=       '<pad>' if pad_token is None else pad_token
                   _lowerCAmelCase    :							Union[str, Any]   			=       '<s>' if bos_token is None else bos_token

            super().__init__(
                do_lower_case=snake_case__      ,	remove_space=snake_case__      ,	keep_accents=snake_case__      ,	bos_token=snake_case__      ,	eos_token=snake_case__      ,	unk_token=snake_case__      ,	pad_token=snake_case__      ,	sp_model_kwargs=self.sp_model_kwargs      ,	**snake_case__      ,	)

            _lowerCAmelCase    :							Union[str, Any]   			=       do_lower_case
            _lowerCAmelCase    :							Optional[int]   			=       remove_space
            _lowerCAmelCase    :							Any   			=       keep_accents
            _lowerCAmelCase    :							Optional[int]   			=       vocab_file

            _lowerCAmelCase    :							Dict   			=       spm.SentencePieceProcessor(**self.sp_model_kwargs   )
            self.sp_model.Load(snake_case__   )

            # Used for whitespace normalization in input texts
            # fmt : off
            _lowerCAmelCase    :							Optional[Any]   			=       {' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
            # fmt : on

            # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
            _lowerCAmelCase    :							Optional[Any]   			=       re.compile(
                F'[{"".join(map(snake_case__      ,	list(range(0      ,	9   )   ) + list(range(11      ,	32   )   ) + list(range(127      ,	160   )   ) + [160, 173, 8203]   )   )}]'   )







     def __getstate__( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							List[str]   			=       self.__dict__.copy()
            _lowerCAmelCase    :							int   			=       None
            return state







     def __setstate__( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       d

            # for backward compatibility
            if not hasattr(self      ,	'sp_model_kwargs'   ):
                   _lowerCAmelCase    :							int   			=       {}

            _lowerCAmelCase    :							List[str]   			=       spm.SentencePieceProcessor(**self.sp_model_kwargs   )
            self.sp_model.Load(self.vocab_file   )







     @property
     # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
     def a	( self   ):

            '''simple docstring'''

            return len(self.sp_model   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       self.non_printing_characters_re.sub(''      ,	snake_case__   )

            # Normalize whitespaces
            _lowerCAmelCase    :							Tuple   			=       ''.join([char if char not in self.whitespaces else ' ' for char in text]   )

            # NFC Unicode normalization
            _lowerCAmelCase    :							Union[str, Any]   			=       unicodedata.normalize('NFC'      ,	snake_case__   )
            return text







     def a	( self      ,	snake_case__      ,	**snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							str   			=       self.preprocess_text(snake_case__   )
            return self.sp_model.encode(snake_case__      ,	out_type=snake_case__   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            return self.sp_model.PieceToId(snake_case__   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            return self.sp_model.IdToPiece(snake_case__   )







     @staticmethod
     def a	( snake_case__   ):

            '''simple docstring'''

            return out_string







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       []
            _lowerCAmelCase    :							Optional[Any]   			=       ''
            _lowerCAmelCase    :							Tuple   			=       False
            for token in tokens:
                   # make sure that special tokens are not decoded using sentencepiece model
                   if token in self.all_special_tokens:
                          # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                          if not prev_is_special:
                                 out_string += " "

                          out_string += self.sp_model.decode(snake_case__   ) + token
                          _lowerCAmelCase    :							Union[str, Any]   			=       True
                          _lowerCAmelCase    :							List[Any]   			=       []
                   else:
                          current_sub_tokens.append(snake_case__   )
                          _lowerCAmelCase    :							List[Any]   			=       False
            out_string += self.sp_model.decode(snake_case__   )

            return out_string







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Union[str, Any]   			=       {self.convert_ids_to_tokens(snake_case__   ): i for i in range(self.vocab_size   )}
            vocab.update(self.added_tokens_encoder   )
            return vocab







     def a	( self      ,	snake_case__      ,	snake_case__ = None   ):

            '''simple docstring'''

            if not os.path.isdir(snake_case__   ):
                   logger.error(F'Vocabulary path ({save_directory}) should be a directory'   )
                   return
            _lowerCAmelCase    :							int   			=       os.path.join(
                snake_case__      ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

            if os.path.abspath(self.vocab_file   ) != os.path.abspath(snake_case__   ) and os.path.isfile(self.vocab_file   ):
                   copyfile(self.vocab_file      ,	snake_case__   )
            elif not os.path.isfile(self.vocab_file   ):
                   with open(snake_case__      ,	'wb'   ) as fi:
                          _lowerCAmelCase    :							Any   			=       self.sp_model.serialized_model_proto()
                          fi.write(snake_case__   )

            return (out_vocab_file,)







     def a	( self      ,	snake_case__      ,	snake_case__ = False   ):

            '''simple docstring'''

            if isinstance(snake_case__      ,	snake_case__   ):
                   _lowerCAmelCase    :							Optional[Any]   			=       self.preprocess_text(snake_case__   )
                   _lowerCAmelCase    :							List[str]   			=       self.sp_model.encode(snake_case__   )
            else:
                   _lowerCAmelCase    :							Tuple   			=       [self.preprocess_text(snake_case__   ) for t in text]
                   _lowerCAmelCase    :							List[str]   			=       self.sp_model.encode(snake_case__   )

            if return_tensors is True or return_tensors == "pt":
                   _lowerCAmelCase    :							int   			=       torch.tensor(snake_case__   )

            return token_ids







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            return self.sp_model.decode(snake_case__   )







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       [F'User: {text}' if is_user else F'Bot: {text}' for is_user, text in conversation.iter_texts()]
            _lowerCAmelCase    :							str   			=       (
                F'{self.eos_token}{self.bos_token}' + F'{self.bos_token}'.join(snake_case__   ) + F'{self.bos_token}Bot:'
            )
            return self.encode(text=snake_case__   )

