


"""simple docstring"""





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
 import torch


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


snake_case__		:     Optional[Any]       				=				logging.get_logger(__name__)
snake_case__		:     Any       				=				{'''vocab_file''': '''spiece.model'''}

snake_case__		:     Union[str, Any]       				=				{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

snake_case__		:     Union[str, Any]       				=				{
    '''AI-Sweden/gpt-sw3-126m''': 2_048,
    '''AI-Sweden/gpt-sw3-350m''': 2_048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2_048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2_048,
    '''AI-Sweden/gpt-sw3-20b''': 2_048,
}



class 		snake_case_(       a__			):
     __UpperCamelCase  =					VOCAB_FILES_NAMES
     __UpperCamelCase  =					PRETRAINED_VOCAB_FILES_MAP
     __UpperCamelCase  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __UpperCamelCase  =					['''input_ids''', '''attention_mask''']






     def __init__(	self  :					List[Any]					,				UpperCamelCase_  :					int					,				UpperCamelCase_  :					Optional[int]=False					,				UpperCamelCase_  :					Tuple=False					,				UpperCamelCase_  :					List[str]=False					,				UpperCamelCase_  :					Any=None					,				UpperCamelCase_  :					Dict=None					,				UpperCamelCase_  :					Any=None					,				UpperCamelCase_  :					List[str]=None					,				UpperCamelCase_  :					Optional[Dict[str, Any]] = None					,				**UpperCamelCase_  :					str					,				):
           lowerCAmelCase					:	Union[str, Any]       =   {} if sp_model_kwargs is None else sp_model_kwargs

           lowerCAmelCase					:	Any       =   kwargs.get('''name_or_path'''				)
           if name_or_path is None:
                 logger.warning(
                     '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
                     ''' you are testing the model, this can safely be ignored'''				)
                 lowerCAmelCase					:	Optional[Any]       =   '''None'''

           # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
           lowerCAmelCase					:	Dict       =   '''<|endoftext|>''' if eos_token is None else eos_token
           lowerCAmelCase					:	int       =   '''<unk>''' if unk_token is None else unk_token
           if "gpt-sw3-7b" in name_or_path:
                 lowerCAmelCase					:	Optional[Any]       =   unk_token if pad_token is None else pad_token
                 lowerCAmelCase					:	Union[str, Any]       =   eos_token if bos_token is None else bos_token
           else:
                 lowerCAmelCase					:	int       =   '''<pad>''' if pad_token is None else pad_token
                 lowerCAmelCase					:	Tuple       =   '''<s>''' if bos_token is None else bos_token

           super().__init__(
               do_lower_case=UpperCamelCase_					,				remove_space=UpperCamelCase_					,				keep_accents=UpperCamelCase_					,				bos_token=UpperCamelCase_					,				eos_token=UpperCamelCase_					,				unk_token=UpperCamelCase_					,				pad_token=UpperCamelCase_					,				sp_model_kwargs=self.sp_model_kwargs					,				**UpperCamelCase_					,				)

           lowerCAmelCase					:	str       =   do_lower_case
           lowerCAmelCase					:	Union[str, Any]       =   remove_space
           lowerCAmelCase					:	Dict       =   keep_accents
           lowerCAmelCase					:	Optional[int]       =   vocab_file

           lowerCAmelCase					:	Optional[Any]       =   spm.SentencePieceProcessor(**self.sp_model_kwargs				)
           self.sp_model.Load(UpperCamelCase_				)

           # Used for whitespace normalization in input texts
           # fmt : off
           lowerCAmelCase					:	int       =   {''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
           # fmt : on

           # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
           lowerCAmelCase					:	List[Any]       =   re.compile(
               F'''[{"".join(map(UpperCamelCase_					,				list(range(0					,				9				)				) + list(range(1_1					,				3_2				)				) + list(range(1_2_7					,				1_6_0				)				) + [1_6_0, 1_7_3, 8_2_0_3]				)				)}]'''				)






     def __getstate__(	self  :					Dict				):
           lowerCAmelCase					:	Tuple       =   self.__dict__.copy()
           lowerCAmelCase					:	Optional[Any]       =   None
           return state






     def __setstate__(	self  :					Union[str, Any]					,				UpperCamelCase_  :					str				):
           lowerCAmelCase					:	Dict       =   d

           # for backward compatibility
           if not hasattr(self					,				'''sp_model_kwargs'''				):
                 lowerCAmelCase					:	Union[str, Any]       =   {}

           lowerCAmelCase					:	int       =   spm.SentencePieceProcessor(**self.sp_model_kwargs				)
           self.sp_model.Load(self.vocab_file				)






     @property
     # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
     def      lowerCamelCase__     (	self  :					Any				):
           return len(self.sp_model				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]					,				UpperCamelCase_  :					str				):
           lowerCAmelCase					:	Any       =   self.non_printing_characters_re.sub(''''''					,				UpperCamelCase_				)

           # Normalize whitespaces
           lowerCAmelCase					:	str       =   ''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]				)

           # NFC Unicode normalization
           lowerCAmelCase					:	Optional[Any]       =   unicodedata.normalize('''NFC'''					,				UpperCamelCase_				)
           return text






     def      lowerCamelCase__     (	self  :					Union[str, Any]					,				UpperCamelCase_  :					str					,				**UpperCamelCase_  :					int				):
           lowerCAmelCase					:	int       =   self.preprocess_text(UpperCamelCase_				)
           return self.sp_model.encode(UpperCamelCase_					,				out_type=UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]					,				UpperCamelCase_  :					str				):
           return self.sp_model.PieceToId(UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					str					,				UpperCamelCase_  :					int				):
           return self.sp_model.IdToPiece(UpperCamelCase_				)






     @staticmethod
     def      lowerCamelCase__     (	UpperCamelCase_  :					str				):
           return out_string






     def      lowerCamelCase__     (	self  :					Dict					,				UpperCamelCase_  :					List[str]				):
           lowerCAmelCase					:	Tuple       =   []
           lowerCAmelCase					:	Optional[int]       =   ''''''
           lowerCAmelCase					:	Union[str, Any]       =   False
           for token in tokens:
                 # make sure that special tokens are not decoded using sentencepiece model
                 if token in self.all_special_tokens:
                       # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                       if not prev_is_special:
                             out_string += " "

                       out_string += self.sp_model.decode(UpperCamelCase_				) + token
                       lowerCAmelCase					:	int       =   True
                       lowerCAmelCase					:	Dict       =   []
                 else:
                       current_sub_tokens.append(UpperCamelCase_				)
                       lowerCAmelCase					:	Optional[Any]       =   False
           out_string += self.sp_model.decode(UpperCamelCase_				)

           return out_string






     def      lowerCamelCase__     (	self  :					List[str]				):
           lowerCAmelCase					:	Dict       =   {self.convert_ids_to_tokens(UpperCamelCase_				): i for i in range(self.vocab_size				)}
           vocab.update(self.added_tokens_encoder				)
           return vocab






     def      lowerCamelCase__     (	self  :					str					,				UpperCamelCase_  :					str					,				UpperCamelCase_  :					Optional[str] = None				):
           if not os.path.isdir(UpperCamelCase_				):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''				)
                 return
           lowerCAmelCase					:	int       =   os.path.join(
               UpperCamelCase_					,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

           if os.path.abspath(self.vocab_file				) != os.path.abspath(UpperCamelCase_				) and os.path.isfile(self.vocab_file				):
                 copyfile(self.vocab_file					,				UpperCamelCase_				)
           elif not os.path.isfile(self.vocab_file				):
                 with open(UpperCamelCase_					,				'''wb'''				) as fi:
                       lowerCAmelCase					:	Any       =   self.sp_model.serialized_model_proto()
                       fi.write(UpperCamelCase_				)

           return (out_vocab_file,)






     def      lowerCamelCase__     (	self  :					Any					,				UpperCamelCase_  :					Union[str, List[str]]					,				UpperCamelCase_  :					Union[str, bool] = False				):

           if isinstance(UpperCamelCase_					,				UpperCamelCase_				):
                 lowerCAmelCase					:	Any       =   self.preprocess_text(UpperCamelCase_				)
                 lowerCAmelCase					:	Optional[int]       =   self.sp_model.encode(UpperCamelCase_				)
           else:
                 lowerCAmelCase					:	Any       =   [self.preprocess_text(UpperCamelCase_				) for t in text]
                 lowerCAmelCase					:	Union[str, Any]       =   self.sp_model.encode(UpperCamelCase_				)

           if return_tensors is True or return_tensors == "pt":
                 lowerCAmelCase					:	List[Any]       =   torch.tensor(UpperCamelCase_				)

           return token_ids






     def      lowerCamelCase__     (	self  :					Union[str, Any]					,				UpperCamelCase_  :					Union[int, List[int]]				):
           return self.sp_model.decode(UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Optional[Any]					,				UpperCamelCase_  :					"Conversation"				):
           lowerCAmelCase					:	Dict       =   [F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
           lowerCAmelCase					:	List[str]       =   (
               F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(UpperCamelCase_				) + F'''{self.bos_token}Bot:'''
           )
           return self.encode(text=UpperCamelCase_				)






"""simple docstring"""





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_big_bird import BigBirdTokenizer
else:
 snake_case__		:     Optional[Any]    				=				None

snake_case__		:     Union[str, Any]    				=				logging.get_logger(__name__)
snake_case__		:     List[str]    				=				{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

snake_case__		:     Any    				=				{
    '''vocab_file''': {
        '''google/bigbird-roberta-base''': '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model''',
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'''
        ),
    },
    '''tokenizer_file''': {
        '''google/bigbird-roberta-base''': (
            '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json'''
        ),
        '''google/bigbird-roberta-large''': (
            '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json'''
        ),
        '''google/bigbird-base-trivia-itc''': (
            '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json'''
        ),
    },
}

snake_case__		:     int    				=				{
    '''google/bigbird-roberta-base''': 4_096,
    '''google/bigbird-roberta-large''': 4_096,
    '''google/bigbird-base-trivia-itc''': 4_096,
}


snake_case__		:     Optional[Any]    				=				'''▁'''



class 		snake_case_(       a__			):
     __UpperCamelCase  =					VOCAB_FILES_NAMES
     __UpperCamelCase  =					PRETRAINED_VOCAB_FILES_MAP
     __UpperCamelCase  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __UpperCamelCase  =					BigBirdTokenizer
     __UpperCamelCase  =					['''input_ids''', '''attention_mask''']
     __UpperCamelCase  =					[]






     def __init__(	self  :					Union[str, Any]					,				UpperCamelCase_  :					str=None					,				UpperCamelCase_  :					Any=None					,				UpperCamelCase_  :					str="<unk>"					,				UpperCamelCase_  :					str="<s>"					,				UpperCamelCase_  :					str="</s>"					,				UpperCamelCase_  :					int="<pad>"					,				UpperCamelCase_  :					List[Any]="[SEP]"					,				UpperCamelCase_  :					Dict="[MASK]"					,				UpperCamelCase_  :					Any="[CLS]"					,				**UpperCamelCase_  :					Any					,				):
           lowerCAmelCase					:	Tuple       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else bos_token
           lowerCAmelCase					:	int       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else eos_token
           lowerCAmelCase					:	List[Any]       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else unk_token
           lowerCAmelCase					:	List[str]       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else pad_token
           lowerCAmelCase					:	Any       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else cls_token
           lowerCAmelCase					:	Tuple       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else sep_token

           # Mask token behave like a normal word, i.e. include the space before it
           lowerCAmelCase					:	Optional[Any]       =   AddedToken(UpperCamelCase_					,				lstrip=UpperCamelCase_					,				rstrip=UpperCamelCase_				) if isinstance(UpperCamelCase_					,				UpperCamelCase_				) else mask_token

           super().__init__(
               UpperCamelCase_					,				tokenizer_file=UpperCamelCase_					,				bos_token=UpperCamelCase_					,				eos_token=UpperCamelCase_					,				unk_token=UpperCamelCase_					,				sep_token=UpperCamelCase_					,				pad_token=UpperCamelCase_					,				cls_token=UpperCamelCase_					,				mask_token=UpperCamelCase_					,				**UpperCamelCase_					,				)

           lowerCAmelCase					:	Optional[int]       =   vocab_file
           lowerCAmelCase					:	Optional[int]       =   False if not self.vocab_file else True






     def      lowerCamelCase__     (	self  :					Union[str, Any]					,				UpperCamelCase_  :					List[int]					,				UpperCamelCase_  :					Optional[List[int]] = None				):
           lowerCAmelCase					:	str       =   [self.sep_token_id]
           lowerCAmelCase					:	Tuple       =   [self.cls_token_id]
           if token_ids_a is None:
                 return cls + token_ids_a + sep
           return cls + token_ids_a + sep + token_ids_a + sep






     def      lowerCamelCase__     (	self  :					Dict					,				UpperCamelCase_  :					List[int]					,				UpperCamelCase_  :					Optional[List[int]] = None					,				UpperCamelCase_  :					bool = False				):

           if already_has_special_tokens:
                 if token_ids_a is not None:
                       raise ValueError(
                           '''You should not supply a second sequence if the provided sequence of '''
                           '''ids is already formatted with special tokens for the model.'''				)
                 return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

           if token_ids_a is None:
                 return [1] + ([0] * len(UpperCamelCase_				)) + [1]
           return [1] + ([0] * len(UpperCamelCase_				)) + [1] + ([0] * len(UpperCamelCase_				)) + [1]






     def      lowerCamelCase__     (	self  :					Dict					,				UpperCamelCase_  :					List[int]					,				UpperCamelCase_  :					Optional[List[int]] = None				):
           lowerCAmelCase					:	Tuple       =   [self.sep_token_id]
           lowerCAmelCase					:	Tuple       =   [self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep				) * [0]
           return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






     def      lowerCamelCase__     (	self  :					Optional[int]					,				UpperCamelCase_  :					str					,				UpperCamelCase_  :					Optional[str] = None				):
           if not self.can_save_slow_tokenizer:
                 raise ValueError(
                     '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                     '''tokenizer.'''				)

           if not os.path.isdir(UpperCamelCase_				):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''				)
                 return
           lowerCAmelCase					:	Optional[int]       =   os.path.join(
               UpperCamelCase_					,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

           if os.path.abspath(self.vocab_file				) != os.path.abspath(UpperCamelCase_				):
                 copyfile(self.vocab_file					,				UpperCamelCase_				)

           return (out_vocab_file,)


