



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a__      :				Tuple   	=			"▁"

a__      :				Union[str, Any]   	=			{"vocab_file": "spiece.model"}

a__      :				List[str]   	=			{
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"}
}

a__      :				Dict   	=			{
    "google/pegasus-xsum": 5_12,
}


a__      :				Dict   	=			logging.get_logger(__name__)



class       UpperCAmelCase__(							lowerCamelCase	):

   '''simple docstring'''




   A    :  Any       				=    VOCAB_FILES_NAMES

   A    :  Tuple       				=    VOCAB_FILES_NAMES
   A    :  Optional[int]       				=    PRETRAINED_VOCAB_FILES_MAP
   A    :  Optional[int]       				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A    :  Optional[Any]       				=    ["input_ids", "attention_mask"]


   def __init__(						self  :			List[Any]			,				lowerCAmelCase  :			Optional[int]			,				lowerCAmelCase  :			Optional[int]="<pad>"			,				lowerCAmelCase  :			Optional[Any]="</s>"			,				lowerCAmelCase  :			Tuple="<unk>"			,				lowerCAmelCase  :			Optional[Any]="<mask_2>"			,				lowerCAmelCase  :			Optional[Any]="<mask_1>"			,				lowerCAmelCase  :			Any=None			,				lowerCAmelCase  :			Optional[int]=1_03			,				lowerCAmelCase  :			Optional[Dict[str, Any]] = None			,				**lowerCAmelCase  :			List[str]			,				)	->				None:




         """simple docstring"""

         lowercase__     =  offset
         if additional_special_tokens is not None:
               if not isinstance(lowerCAmelCase			,				lowerCAmelCase):
                     raise TypeError(
                         f'''additional_special_tokens should be of type {type(lowerCAmelCase)}, but is'''
                         f''' {type(lowerCAmelCase)}''')

               lowercase__     =  (
                   ([mask_token_sent] + additional_special_tokens)
                   if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                   else additional_special_tokens
               )
               # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
               additional_special_tokens_extended += [
                   f'''<unk_{i}>''' for i in range(len(lowerCAmelCase)			,				self.offset - 1)
               ]

               if len(set(lowerCAmelCase)) != len(lowerCAmelCase):
                     raise ValueError(
                         'Please make sure that the provided additional_special_tokens do not contain an incorrectly'
                         f''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.''')
               lowercase__     =  additional_special_tokens_extended
         else:
               lowercase__     =  [mask_token_sent] if mask_token_sent is not None else []
               additional_special_tokens += [f'''<unk_{i}>''' for i in range(2			,				self.offset)]

         lowercase__     =  {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             eos_token=lowerCAmelCase			,				unk_token=lowerCAmelCase			,				mask_token=lowerCAmelCase			,				pad_token=lowerCAmelCase			,				mask_token_sent=lowerCAmelCase			,				offset=lowerCAmelCase			,				additional_special_tokens=lowerCAmelCase			,				sp_model_kwargs=self.sp_model_kwargs			,				**lowerCAmelCase			,				)
         lowercase__     =  mask_token_sent
         lowercase__     =  vocab_file
         lowercase__     =  spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(lowerCAmelCase)

         # add special tokens to encoder dict
         lowercase__     =  {
             0: self.pad_token,
             1: self.eos_token,
         }

         if self.mask_token_sent is not None:
               self.encoder.update(
                   {
                       2: self.mask_token_sent,
                       3: self.mask_token,
                   })

         if self.offset > 0:
               # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
               # mask_token_sent is already added to list -> so start at 1
               self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1			,				self.offset - 1)})

         lowercase__     =  {v: k for k, v in self.encoder.items()}


   @property
   def 			UpperCAmelCase		(						self  :			Optional[int])	->				int:




         """simple docstring"""

         return len(self.sp_model) + self.offset


   def 			UpperCAmelCase		(						self  :			Dict)	->				Dict[str, int]:




         """simple docstring"""

         lowercase__     =  {self.convert_ids_to_tokens(lowerCAmelCase): i for i in range(self.vocab_size)}
         vocab.update(self.added_tokens_encoder)
         return vocab


   def __getstate__(						self  :			List[str])	->				List[str]:




         """simple docstring"""

         lowercase__     =  self.__dict__.copy()
         lowercase__     =  None
         return state


   def __setstate__(						self  :			Optional[Any]			,				lowerCAmelCase  :			int)	->				Tuple:




         """simple docstring"""

         lowercase__     =  d

         # for backward compatibility
         if not hasattr(self			,				'sp_model_kwargs'):
               lowercase__     =  {}

         lowercase__     =  spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(self.vocab_file)


   def 			UpperCAmelCase		(						self  :			Tuple			,				lowerCAmelCase  :			str)	->				List[str]:




         """simple docstring"""

         return self.sp_model.encode(lowerCAmelCase			,				out_type=lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			Optional[int]			,				lowerCAmelCase  :			str)	->				int:




         """simple docstring"""

         if token in self.decoder:
               return self.decoder[token]
         elif token in self.added_tokens_decoder:
               return self.added_tokens_decoder[token]
         lowercase__     =  self.sp_model.piece_to_id(lowerCAmelCase)
         return sp_id + self.offset


   def 			UpperCAmelCase		(						self  :			List[str]			,				lowerCAmelCase  :			int)	->				str:




         """simple docstring"""

         if index in self.encoder:
               return self.encoder[index]
         elif index in self.added_tokens_encoder:
               return self.added_tokens_encoder[index]
         else:
               lowercase__     =  self.sp_model.IdToPiece(index - self.offset)
         return token


   def 			UpperCAmelCase		(						self  :			Optional[Any]			,				lowerCAmelCase  :			Optional[int])	->				int:




         """simple docstring"""

         lowercase__     =  []
         lowercase__     =  ''
         for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                     out_string += self.sp_model.decode(lowerCAmelCase) + token
                     lowercase__     =  []
               else:
                     current_sub_tokens.append(lowerCAmelCase)
         out_string += self.sp_model.decode(lowerCAmelCase)
         return out_string.strip()


   def 			UpperCAmelCase		(						self  :			Tuple			,				lowerCAmelCase  :			int=False)	->				List[Any]:




         """simple docstring"""

         return 1


   def 			UpperCAmelCase		(						self  :			str			,				lowerCAmelCase  :			Dict)	->				List[Any]:




         """simple docstring"""

         lowercase__     =  set(self.all_special_ids)  # call it once instead of inside list comp
         all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special

         return [1 if x in all_special_ids else 0 for x in seq]


   def 			UpperCAmelCase		(						self  :			List[Any]			,				lowerCAmelCase  :			List			,				lowerCAmelCase  :			Optional[List] = None			,				lowerCAmelCase  :			bool = False)	->				List[int]:




         """simple docstring"""

         if already_has_special_tokens:
               return self._special_token_mask(lowerCAmelCase)
         elif token_ids_a is None:
               return self._special_token_mask(lowerCAmelCase) + [1]
         else:
               return self._special_token_mask(token_ids_a + token_ids_a) + [1]


   def 			UpperCAmelCase		(						self  :			Tuple			,				lowerCAmelCase  :			Dict			,				lowerCAmelCase  :			Union[str, Any]=None)	->				List[int]:




         """simple docstring"""

         if token_ids_a is None:
               return token_ids_a + [self.eos_token_id]
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return token_ids_a + token_ids_a + [self.eos_token_id]



   def 			UpperCAmelCase		(						self  :			Any			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			Optional[str] = None)	->				Tuple[str]:




         """simple docstring"""

         if not os.path.isdir(lowerCAmelCase):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
               return
         lowercase__     =  os.path.join(
             lowerCAmelCase			,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

         if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCAmelCase) and os.path.isfile(self.vocab_file):
               copyfile(self.vocab_file			,				lowerCAmelCase)
         elif not os.path.isfile(self.vocab_file):
               with open(lowerCAmelCase			,				'wb') as fi:
                     lowercase__     =  self.sp_model.serialized_model_proto()
                     fi.write(lowerCAmelCase)

         return (out_vocab_file,)





import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin



class       UpperCAmelCase__(							unittest.TestCase     ,							lowerCamelCase	):

   '''simple docstring'''




   def 			UpperCAmelCase		(						self  :			List[str])	->				Any:




         """simple docstring"""

         lowercase__     =  load_tool('text-classification')
         self.tool.setup()
         lowercase__     =  load_tool('text-classification'			,				remote=lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			Any)	->				Tuple:




         """simple docstring"""

         lowercase__     =  self.tool('That\'s quite cool'			,				['positive', 'negative'])
         self.assertEqual(lowerCAmelCase			,				'positive')


   def 			UpperCAmelCase		(						self  :			int)	->				Optional[int]:




         """simple docstring"""

         lowercase__     =  self.remote_tool('That\'s quite cool'			,				['positive', 'negative'])
         self.assertEqual(lowerCAmelCase			,				'positive')


   def 			UpperCAmelCase		(						self  :			Optional[Any])	->				Union[str, Any]:




         """simple docstring"""

         lowercase__     =  self.tool(text='That\'s quite cool'			,				labels=['positive', 'negative'])
         self.assertEqual(lowerCAmelCase			,				'positive')



   def 			UpperCAmelCase		(						self  :			Any)	->				Any:




         """simple docstring"""

         lowercase__     =  self.remote_tool(text='That\'s quite cool'			,				labels=['positive', 'negative'])
         self.assertEqual(lowerCAmelCase			,				'positive')
