



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a__      :				int   	=			"▁"

a__      :				Dict   	=			{"vocab_file": "spiece.model"}

a__      :				int   	=			{
    "vocab_file": {"google/pegasus-xsum": "https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"}
}

a__      :				List[str]   	=			{
    "google/pegasus-xsum": 5_12,
}


a__      :				Any   	=			logging.get_logger(__name__)



class       UpperCAmelCase__(							__UpperCAmelCase	):

			'''simple docstring'''




			A    :  List[Any]       				=    VOCAB_FILES_NAMES

			A    :  List[str]       				=    VOCAB_FILES_NAMES
			A    :  int       				=    PRETRAINED_VOCAB_FILES_MAP
			A    :  List[Any]       				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A    :  List[Any]       				=    ["input_ids", "attention_mask"]


			def __init__(						self  :			Tuple			,				lowerCAmelCase  :			List[Any]			,				lowerCAmelCase  :			str="<pad>"			,				lowerCAmelCase  :			int="</s>"			,				lowerCAmelCase  :			Tuple="<unk>"			,				lowerCAmelCase  :			List[Any]="<mask_2>"			,				lowerCAmelCase  :			Union[str, Any]="<mask_1>"			,				lowerCAmelCase  :			List[Any]=None			,				lowerCAmelCase  :			int=1_03			,				lowerCAmelCase  :			Optional[Dict[str, Any]] = None			,				**lowerCAmelCase  :			List[str]			,				)	->				Optional[int]:




									"""simple docstring"""

									lowercase__     =  offset
									if additional_special_tokens is not None:
															if not isinstance(_lowerCamelCase			,				_lowerCamelCase):
																					raise TypeError(
																					    f'''additional_special_tokens should be of type {type(_lowerCamelCase)}, but is'''
																					    f''' {type(_lowerCamelCase)}''')

															lowercase__     =  (
															    ([mask_token_sent] + additional_special_tokens)
															    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
															    else additional_special_tokens
															)
															# fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
															additional_special_tokens_extended += [
															    f'''<unk_{i}>''' for i in range(len(_lowerCamelCase)			,				self.offset - 1)
															]

															if len(set(_lowerCamelCase)) != len(_lowerCamelCase):
																					raise ValueError(
																					    'Please make sure that the provided additional_special_tokens do not contain an incorrectly'
																					    f''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.''')
															lowercase__     =  additional_special_tokens_extended
									else:
															lowercase__     =  [mask_token_sent] if mask_token_sent is not None else []
															additional_special_tokens += [f'''<unk_{i}>''' for i in range(2			,				self.offset)]

									lowercase__     =  {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    eos_token=_lowerCamelCase			,				unk_token=_lowerCamelCase			,				mask_token=_lowerCamelCase			,				pad_token=_lowerCamelCase			,				mask_token_sent=_lowerCamelCase			,				offset=_lowerCamelCase			,				additional_special_tokens=_lowerCamelCase			,				sp_model_kwargs=self.sp_model_kwargs			,				**_lowerCamelCase			,				)
									lowercase__     =  mask_token_sent
									lowercase__     =  vocab_file
									lowercase__     =  spm.SentencePieceProcessor(**self.sp_model_kwargs)
									self.sp_model.Load(_lowerCamelCase)

									# add special tokens to encoder dict
									lowercase__     =  {
									    0: self.pad_token,
									    1: self.eos_token,
									}

									if self.mask_token_sent is not None:
															self.encoder.update(
															    {
															        2: self.mask_token_sent,
															        3: self.mask_token,
															    })

									if self.offset > 0:
															# entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
															# mask_token_sent is already added to list -> so start at 1
															self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1			,				self.offset - 1)})

									lowercase__     =  {v: k for k, v in self.encoder.items()}


			@property
			def 			UpperCAmelCase		(						self  :			Tuple)	->				Tuple:




									"""simple docstring"""

									return len(self.sp_model) + self.offset


			def 			UpperCAmelCase		(						self  :			Optional[Any])	->				Dict:




									"""simple docstring"""

									lowercase__     =  {self.convert_ids_to_tokens(_lowerCamelCase): i for i in range(self.vocab_size)}
									vocab.update(self.added_tokens_encoder)
									return vocab


			def __getstate__(						self  :			Tuple)	->				str:




									"""simple docstring"""

									lowercase__     =  self.__dict__.copy()
									lowercase__     =  None
									return state


			def __setstate__(						self  :			str			,				lowerCAmelCase  :			List[str])	->				List[str]:




									"""simple docstring"""

									lowercase__     =  d

									# for backward compatibility
									if not hasattr(self			,				'sp_model_kwargs'):
															lowercase__     =  {}

									lowercase__     =  spm.SentencePieceProcessor(**self.sp_model_kwargs)
									self.sp_model.Load(self.vocab_file)


			def 			UpperCAmelCase		(						self  :			List[str]			,				lowerCAmelCase  :			str)	->				Union[str, Any]:




									"""simple docstring"""

									return self.sp_model.encode(_lowerCamelCase			,				out_type=_lowerCamelCase)


			def 			UpperCAmelCase		(						self  :			List[Any]			,				lowerCAmelCase  :			str)	->				Tuple:




									"""simple docstring"""

									if token in self.decoder:
															return self.decoder[token]
									elif token in self.added_tokens_decoder:
															return self.added_tokens_decoder[token]
									lowercase__     =  self.sp_model.piece_to_id(_lowerCamelCase)
									return sp_id + self.offset


			def 			UpperCAmelCase		(						self  :			Optional[Any]			,				lowerCAmelCase  :			int)	->				Dict:




									"""simple docstring"""

									if index in self.encoder:
															return self.encoder[index]
									elif index in self.added_tokens_encoder:
															return self.added_tokens_encoder[index]
									else:
															lowercase__     =  self.sp_model.IdToPiece(index - self.offset)
									return token


			def 			UpperCAmelCase		(						self  :			Optional[Any]			,				lowerCAmelCase  :			List[Any])	->				Any:




									"""simple docstring"""

									lowercase__     =  []
									lowercase__     =  ''
									for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																					out_string += self.sp_model.decode(_lowerCamelCase) + token
																					lowercase__     =  []
															else:
																					current_sub_tokens.append(_lowerCamelCase)
									out_string += self.sp_model.decode(_lowerCamelCase)
									return out_string.strip()


			def 			UpperCAmelCase		(						self  :			List[str]			,				lowerCAmelCase  :			List[Any]=False)	->				int:




									"""simple docstring"""

									return 1


			def 			UpperCAmelCase		(						self  :			Any			,				lowerCAmelCase  :			Tuple)	->				str:




									"""simple docstring"""

									lowercase__     =  set(self.all_special_ids)  # call it once instead of inside list comp
									all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special

									return [1 if x in all_special_ids else 0 for x in seq]


			def 			UpperCAmelCase		(						self  :			int			,				lowerCAmelCase  :			List			,				lowerCAmelCase  :			Optional[List] = None			,				lowerCAmelCase  :			bool = False)	->				Any:




									"""simple docstring"""

									if already_has_special_tokens:
															return self._special_token_mask(_lowerCamelCase)
									elif token_ids_a is None:
															return self._special_token_mask(_lowerCamelCase) + [1]
									else:
															return self._special_token_mask(token_ids_a + token_ids_a) + [1]


			def 			UpperCAmelCase		(						self  :			Optional[Any]			,				lowerCAmelCase  :			Optional[Any]			,				lowerCAmelCase  :			Tuple=None)	->				Optional[Any]:




									"""simple docstring"""

									if token_ids_a is None:
															return token_ids_a + [self.eos_token_id]
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return token_ids_a + token_ids_a + [self.eos_token_id]



			def 			UpperCAmelCase		(						self  :			Optional[int]			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			Optional[str] = None)	->				List[Any]:




									"""simple docstring"""

									if not os.path.isdir(_lowerCamelCase):
															logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
															return
									lowercase__     =  os.path.join(
									    _lowerCamelCase			,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

									if os.path.abspath(self.vocab_file) != os.path.abspath(_lowerCamelCase) and os.path.isfile(self.vocab_file):
															copyfile(self.vocab_file			,				_lowerCamelCase)
									elif not os.path.isfile(self.vocab_file):
															with open(_lowerCamelCase			,				'wb') as fi:
																					lowercase__     =  self.sp_model.serialized_model_proto()
																					fi.write(_lowerCamelCase)

									return (out_vocab_file,)





# Imports
import numpy as np



class       UpperCAmelCase__:

   '''simple docstring'''




   def __init__(						self  :			Any			,				lowerCAmelCase  :			Dict=None			,				lowerCAmelCase  :			List[Any]=None			,				lowerCAmelCase  :			List[Any]=None			,				lowerCAmelCase  :			List[str]=None			,				lowerCAmelCase  :			List[str]=None)	->				Dict:




         """simple docstring"""

         self.set_matricies(red=lowerCAmelCase			,				green=lowerCAmelCase			,				blue=lowerCAmelCase			,				red_edge=lowerCAmelCase			,				nir=lowerCAmelCase)


   def 			UpperCAmelCase		(						self  :			Dict			,				lowerCAmelCase  :			Dict=None			,				lowerCAmelCase  :			Union[str, Any]=None			,				lowerCAmelCase  :			Tuple=None			,				lowerCAmelCase  :			str=None			,				lowerCAmelCase  :			str=None)	->				int:




         """simple docstring"""

         if red is not None:
               lowercase__     =  red
         if green is not None:
               lowercase__     =  green
         if blue is not None:
               lowercase__     =  blue
         if red_edge is not None:
               lowercase__     =  red_edge
         if nir is not None:
               lowercase__     =  nir
         return True


   def 			UpperCAmelCase		(						self  :			Optional[int]			,				lowerCAmelCase  :			Union[str, Any]=""			,				lowerCAmelCase  :			Tuple=None			,				lowerCAmelCase  :			Optional[Any]=None			,				lowerCAmelCase  :			Optional[Any]=None			,				lowerCAmelCase  :			List[Any]=None			,				lowerCAmelCase  :			Dict=None)	->				Union[str, Any]:




         """simple docstring"""

         self.set_matricies(red=lowerCAmelCase			,				green=lowerCAmelCase			,				blue=lowerCAmelCase			,				red_edge=lowerCAmelCase			,				nir=lowerCAmelCase)
         lowercase__     =  {
             'ARVI2': self.arvaa,
             'CCCI': self.ccci,
             'CVI': self.cvi,
             'GLI': self.gli,
             'NDVI': self.ndvi,
             'BNDVI': self.bndvi,
             'redEdgeNDVI': self.red_edge_ndvi,
             'GNDVI': self.gndvi,
             'GBNDVI': self.gbndvi,
             'GRNDVI': self.grndvi,
             'RBNDVI': self.rbndvi,
             'PNDVI': self.pndvi,
             'ATSAVI': self.atsavi,
             'BWDRVI': self.bwdrvi,
             'CIgreen': self.ci_green,
             'CIrededge': self.ci_rededge,
             'CI': self.ci,
             'CTVI': self.ctvi,
             'GDVI': self.gdvi,
             'EVI': self.evi,
             'GEMI': self.gemi,
             'GOSAVI': self.gosavi,
             'GSAVI': self.gsavi,
             'Hue': self.hue,
             'IVI': self.ivi,
             'IPVI': self.ipvi,
             'I': self.i,
             'RVI': self.rvi,
             'MRVI': self.mrvi,
             'MSAVI': self.m_savi,
             'NormG': self.norm_g,
             'NormNIR': self.norm_nir,
             'NormR': self.norm_r,
             'NGRDI': self.ngrdi,
             'RI': self.ri,
             'S': self.s,
             'IF': self._if,
             'DVI': self.dvi,
             'TVI': self.tvi,
             'NDRE': self.ndre,
         }

         try:
               return funcs[index]()
         except KeyError:
               print('Index not in the list!')
               return False


   def 			UpperCAmelCase		(						self  :			Optional[int])	->				List[str]:




         """simple docstring"""

         return -0.18 + (1.17 * ((self.nir - self.red) / (self.nir + self.red)))


   def 			UpperCAmelCase		(						self  :			int)	->				Any:




         """simple docstring"""

         return ((self.nir - self.redEdge) / (self.nir + self.redEdge)) / (
             (self.nir - self.red) / (self.nir + self.red)
         )


   def 			UpperCAmelCase		(						self  :			str)	->				Optional[int]:




         """simple docstring"""

         return self.nir * (self.red / (self.green**2))


   def 			UpperCAmelCase		(						self  :			List[str])	->				Optional[int]:




         """simple docstring"""

         return (2 * self.green - self.red - self.blue) / (
             2 * self.green + self.red + self.blue
         )


   def 			UpperCAmelCase		(						self  :			Tuple)	->				Any:




         """simple docstring"""

         return (self.nir - self.red) / (self.nir + self.red)


   def 			UpperCAmelCase		(						self  :			Union[str, Any])	->				str:




         """simple docstring"""

         return (self.nir - self.blue) / (self.nir + self.blue)


   def 			UpperCAmelCase		(						self  :			List[Any])	->				Optional[int]:




         """simple docstring"""

         return (self.redEdge - self.red) / (self.redEdge + self.red)


   def 			UpperCAmelCase		(						self  :			Optional[Any])	->				Optional[int]:




         """simple docstring"""

         return (self.nir - self.green) / (self.nir + self.green)


   def 			UpperCAmelCase		(						self  :			Dict)	->				int:




         """simple docstring"""

         return (self.nir - (self.green + self.blue)) / (
             self.nir + (self.green + self.blue)
         )


   def 			UpperCAmelCase		(						self  :			Union[str, Any])	->				Optional[int]:




         """simple docstring"""

         return (self.nir - (self.green + self.red)) / (
             self.nir + (self.green + self.red)
         )


   def 			UpperCAmelCase		(						self  :			Optional[Any])	->				Dict:




         """simple docstring"""

         return (self.nir - (self.blue + self.red)) / (self.nir + (self.blue + self.red))


   def 			UpperCAmelCase		(						self  :			Any)	->				Union[str, Any]:




         """simple docstring"""

         return (self.nir - (self.green + self.red + self.blue)) / (
             self.nir + (self.green + self.red + self.blue)
         )


   def 			UpperCAmelCase		(						self  :			Optional[int]			,				lowerCAmelCase  :			List[Any]=0.08			,				lowerCAmelCase  :			Optional[int]=1.22			,				lowerCAmelCase  :			int=0.03)	->				List[Any]:




         """simple docstring"""

         return a * (
             (self.nir - a * self.red - b)
             / (a * self.nir + self.red - a * b + x * (1 + a**2))
         )


   def 			UpperCAmelCase		(						self  :			Tuple)	->				Any:




         """simple docstring"""

         return (0.1 * self.nir - self.blue) / (0.1 * self.nir + self.blue)


   def 			UpperCAmelCase		(						self  :			int)	->				Tuple:




         """simple docstring"""

         return (self.nir / self.green) - 1


   def 			UpperCAmelCase		(						self  :			Any)	->				str:




         """simple docstring"""

         return (self.nir / self.redEdge) - 1


   def 			UpperCAmelCase		(						self  :			Any)	->				List[str]:




         """simple docstring"""

         return (self.red - self.blue) / self.red


   def 			UpperCAmelCase		(						self  :			Any)	->				Optional[int]:




         """simple docstring"""

         lowercase__     =  self.ndvi()
         return ((ndvi + 0.5) / (abs(ndvi + 0.5))) * (abs(ndvi + 0.5) ** (1 / 2))


   def 			UpperCAmelCase		(						self  :			List[Any])	->				str:




         """simple docstring"""

         return self.nir - self.green


   def 			UpperCAmelCase		(						self  :			Tuple)	->				List[Any]:




         """simple docstring"""

         return 2.5 * (
             (self.nir - self.red) / (self.nir + 6 * self.red - 7.5 * self.blue + 1)
         )


   def 			UpperCAmelCase		(						self  :			Any)	->				Union[str, Any]:




         """simple docstring"""

         lowercase__     =  (2 * (self.nir**2 - self.red**2) + 1.5 * self.nir + 0.5 * self.red) / (
             self.nir + self.red + 0.5
         )
         return n * (1 - 0.25 * n) - (self.red - 0.1_25) / (1 - self.red)


   def 			UpperCAmelCase		(						self  :			int			,				lowerCAmelCase  :			int=0.16)	->				Dict:




         """simple docstring"""

         return (self.nir - self.green) / (self.nir + self.green + y)


   def 			UpperCAmelCase		(						self  :			str			,				lowerCAmelCase  :			Optional[int]=0.5)	->				Union[str, Any]:




         """simple docstring"""

         return ((self.nir - self.green) / (self.nir + self.green + n)) * (1 + n)


   def 			UpperCAmelCase		(						self  :			str)	->				int:




         """simple docstring"""

         return np.arctan(
             ((2 * self.red - self.green - self.blue) / 30.5) * (self.green - self.blue))


   def 			UpperCAmelCase		(						self  :			Union[str, Any]			,				lowerCAmelCase  :			int=None			,				lowerCAmelCase  :			List[str]=None)	->				Tuple:




         """simple docstring"""

         return (self.nir - b) / (a * self.red)


   def 			UpperCAmelCase		(						self  :			int)	->				Dict:




         """simple docstring"""

         return (self.nir / ((self.nir + self.red) / 2)) * (self.ndvi() + 1)


   def 			UpperCAmelCase		(						self  :			Optional[int])	->				Optional[Any]:




         """simple docstring"""

         return (self.red + self.green + self.blue) / 30.5


   def 			UpperCAmelCase		(						self  :			int)	->				str:




         """simple docstring"""

         return self.nir / self.red


   def 			UpperCAmelCase		(						self  :			Optional[int])	->				Optional[Any]:




         """simple docstring"""

         return (self.rvi() - 1) / (self.rvi() + 1)


   def 			UpperCAmelCase		(						self  :			Optional[int])	->				Optional[int]:




         """simple docstring"""

         return (
             (2 * self.nir + 1)
             - ((2 * self.nir + 1) ** 2 - 8 * (self.nir - self.red)) ** (1 / 2)
         ) / 2


   def 			UpperCAmelCase		(						self  :			Tuple)	->				Any:




         """simple docstring"""

         return self.green / (self.nir + self.red + self.green)


   def 			UpperCAmelCase		(						self  :			Any)	->				Optional[Any]:




         """simple docstring"""

         return self.nir / (self.nir + self.red + self.green)


   def 			UpperCAmelCase		(						self  :			List[Any])	->				Dict:




         """simple docstring"""

         return self.red / (self.nir + self.red + self.green)


   def 			UpperCAmelCase		(						self  :			Optional[Any])	->				Any:




         """simple docstring"""

         return (self.green - self.red) / (self.green + self.red)


   def 			UpperCAmelCase		(						self  :			Dict)	->				Tuple:




         """simple docstring"""

         return (self.red - self.green) / (self.red + self.green)


   def 			UpperCAmelCase		(						self  :			str)	->				int:




         """simple docstring"""

         lowercase__     =  np.max([np.max(self.red), np.max(self.green), np.max(self.blue)])
         lowercase__     =  np.min([np.min(self.red), np.min(self.green), np.min(self.blue)])
         return (max_value - min_value) / max_value


   def 			UpperCAmelCase		(						self  :			Optional[int])	->				Tuple:




         """simple docstring"""

         return (2 * self.red - self.green - self.blue) / (self.green - self.blue)


   def 			UpperCAmelCase		(						self  :			int)	->				Optional[Any]:




         """simple docstring"""

         return self.nir / self.red


   def 			UpperCAmelCase		(						self  :			Dict)	->				Dict:




         """simple docstring"""

         return (self.ndvi() + 0.5) ** (1 / 2)



   def 			UpperCAmelCase		(						self  :			str)	->				List[Any]:




         """simple docstring"""

         return (self.nir - self.redEdge) / (self.nir + self.redEdge)
