
"""simple docstring"""




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A__      :	List[str]          =    logging.get_logger(__name__)
A__      :	Optional[int]          =    {'vocab_file': 'spiece.model'}

A__      :	List[str]          =    {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

A__      :	List[str]          =    {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

A__      :	int          =    '▁'







class   _lowercase						(       snake_case__				):


     '''simple docstring'''





     _A								=							VOCAB_FILES_NAMES
     _A								=							PRETRAINED_VOCAB_FILES_MAP
     _A								=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



     def __init__(      self ,			__UpperCamelCase ,			__UpperCamelCase=True ,			__UpperCamelCase=True ,			__UpperCamelCase=False ,			__UpperCamelCase="[CLS]" ,			__UpperCamelCase="[SEP]" ,			__UpperCamelCase="<unk>" ,			__UpperCamelCase="[SEP]" ,			__UpperCamelCase="<pad>" ,			__UpperCamelCase="[CLS]" ,			__UpperCamelCase="[MASK]" ,			__UpperCamelCase = None ,			**__UpperCamelCase ,			)->					None:
          UpperCAmelCase__							: List[Any] 							=						(
              AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_ ,			normalized=UpperCAmelCase_  )
              if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_  )
              else mask_token
          )

          UpperCAmelCase__							: Tuple 							=						{} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              do_lower_case=UpperCAmelCase_ ,			remove_space=UpperCAmelCase_ ,			keep_accents=UpperCAmelCase_ ,			bos_token=UpperCAmelCase_ ,			eos_token=UpperCAmelCase_ ,			unk_token=UpperCAmelCase_ ,			sep_token=UpperCAmelCase_ ,			pad_token=UpperCAmelCase_ ,			cls_token=UpperCAmelCase_ ,			mask_token=UpperCAmelCase_ ,			sp_model_kwargs=self.sp_model_kwargs ,			**UpperCAmelCase_ ,			)

          UpperCAmelCase__							: Optional[int] 							=						do_lower_case
          UpperCAmelCase__							: Optional[int] 							=						remove_space
          UpperCAmelCase__							: int 							=						keep_accents
          UpperCAmelCase__							: Tuple 							=						vocab_file

          UpperCAmelCase__							: int 							=						spm.SentencePieceProcessor(**self.sp_model_kwargs  )
          self.sp_model.Load(UpperCAmelCase_  )



     @property
     def  lowerCAmelCase__       (      self  )->					Dict:
          return len(self.sp_model  )



     def  lowerCAmelCase__       (      self  )->					List[Any]:
          UpperCAmelCase__							: Tuple 							=						{self.convert_ids_to_tokens(UpperCAmelCase_  ): i for i in range(self.vocab_size  )}
          vocab.update(self.added_tokens_encoder  )
          return vocab



     def __getstate__(      self  )->					List[str]:
          UpperCAmelCase__							: List[str] 							=						self.__dict__.copy()
          UpperCAmelCase__							: Optional[Any] 							=						None
          return state



     def __setstate__(      self ,			__UpperCamelCase  )->					Optional[int]:
          UpperCAmelCase__							: Union[str, Any] 							=						d

          # for backward compatibility
          if not hasattr(self ,			"sp_model_kwargs"  ):
               UpperCAmelCase__							: Union[str, Any] 							=						{}

          UpperCAmelCase__							: Optional[int] 							=						spm.SentencePieceProcessor(**self.sp_model_kwargs  )
          self.sp_model.Load(self.vocab_file  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					str:



          if self.remove_space:
               UpperCAmelCase__							: Optional[Any] 							=						" ".join(inputs.strip().split()  )
          else:
               UpperCAmelCase__							: Any 							=						inputs
          UpperCAmelCase__							: Optional[Any] 							=						outputs.replace("``" ,			"\""  ).replace("''" ,			"\""  )

          if not self.keep_accents:
               UpperCAmelCase__							: int 							=						unicodedata.normalize("NFKD" ,			UpperCAmelCase_  )
               UpperCAmelCase__							: Dict 							=						"".join([c for c in outputs if not unicodedata.combining(UpperCAmelCase_  )]  )
          if self.do_lower_case:
               UpperCAmelCase__							: Dict 							=						outputs.lower()

          return outputs



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					List[str]:
          UpperCAmelCase__							: Union[str, Any] 							=						self.preprocess_text(UpperCAmelCase_  )
          UpperCAmelCase__							: Union[str, Any] 							=						self.sp_model.encode(UpperCAmelCase_ ,			out_type=UpperCAmelCase_  )
          UpperCAmelCase__							: Tuple 							=						[]
          for piece in pieces:
               if len(UpperCAmelCase_  ) > 1 and piece[-1] == str(","  ) and piece[-2].isdigit():
                    UpperCAmelCase__							: List[str] 							=						self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase_ ,			""  )  )
                    if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                         if len(cur_pieces[0]  ) == 1:
                              UpperCAmelCase__							: Any 							=						cur_pieces[1:]
                         else:
                              UpperCAmelCase__							: str 							=						cur_pieces[0][1:]
                    cur_pieces.append(piece[-1]  )
                    new_pieces.extend(UpperCAmelCase_  )
               else:
                    new_pieces.append(UpperCAmelCase_  )

          return new_pieces



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Dict:
          return self.sp_model.PieceToId(UpperCAmelCase_  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Optional[int]:
          return self.sp_model.IdToPiece(UpperCAmelCase_  )



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase  )->					Any:
          UpperCAmelCase__							: str 							=						[]
          UpperCAmelCase__							: int 							=						""
          UpperCAmelCase__							: Any 							=						False
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    if not prev_is_special:
                         out_string += " "
                    out_string += self.sp_model.decode(UpperCAmelCase_  ) + token
                    UpperCAmelCase__							: int 							=						True
                    UpperCAmelCase__							: Optional[int] 							=						[]
               else:
                    current_sub_tokens.append(UpperCAmelCase_  )
                    UpperCAmelCase__							: Optional[Any] 							=						False
          out_string += self.sp_model.decode(UpperCAmelCase_  )
          return out_string.strip()



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase = None  )->					List[int]:
          UpperCAmelCase__							: Dict 							=						[self.sep_token_id]
          UpperCAmelCase__							: Optional[int] 							=						[self.cls_token_id]
          if token_ids_a is None:
               return cls + token_ids_a + sep
          return cls + token_ids_a + sep + token_ids_a + sep



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase = None ,			__UpperCamelCase = False  )->					List[int]:



          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=UpperCAmelCase_ ,			token_ids_a=UpperCAmelCase_ ,			already_has_special_tokens=UpperCAmelCase_  )

          if token_ids_a is not None:
               return [1] + ([0] * len(UpperCAmelCase_  )) + [1] + ([0] * len(UpperCAmelCase_  )) + [1]
          return [1] + ([0] * len(UpperCAmelCase_  )) + [1]



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase = None  )->					List[int]:
          UpperCAmelCase__							: int 							=						[self.sep_token_id]
          UpperCAmelCase__							: Tuple 							=						[self.cls_token_id]

          if token_ids_a is None:
               return len(cls + token_ids_a + sep  ) * [0]
          return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]



     def  lowerCAmelCase__       (      self ,			__UpperCamelCase ,			__UpperCamelCase = None  )->					Tuple[str]:



          if not os.path.isdir(UpperCAmelCase_  ):
               logger.error(F"Vocabulary path ({save_directory}) should be a directory"  )
               return
          UpperCAmelCase__							: str 							=						os.path.join(
              UpperCAmelCase_ ,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

          if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCAmelCase_  ) and os.path.isfile(self.vocab_file  ):
               copyfile(self.vocab_file ,			UpperCAmelCase_  )
          elif not os.path.isfile(self.vocab_file  ):
               with open(UpperCAmelCase_ ,			"wb"  ) as fi:
                    UpperCAmelCase__							: Dict 							=						self.sp_model.serialized_model_proto()
                    fi.write(UpperCAmelCase_  )

          return (out_vocab_file,)





"""simple docstring"""




def 							a__ (				lowerCAmelCase	:       list       ,    lowerCAmelCase	:       list		):






     '''simple docstring'''
     _validate_point(lowerCAmelCase		)
     _validate_point(lowerCAmelCase		)
     if len(lowerCAmelCase		) != len(lowerCAmelCase		):
          raise ValueError("Both points must be in the same n-dimensional space"		)

     return float(sum(abs(a - b		) for a, b in zip(lowerCAmelCase       ,    lowerCAmelCase		)		)		)



def 							a__ (				lowerCAmelCase	:       list[float]		):






     '''simple docstring'''
     if point:
          if isinstance(lowerCAmelCase       ,    lowerCAmelCase		):
               for item in point:
                    if not isinstance(lowerCAmelCase       ,    (int, float)		):
                         UpperCAmelCase__							: Tuple 							=						(
                             "Expected a list of numbers as input, found "
                             F"{type(lowerCAmelCase		).__name__}"
                         )
                         raise TypeError(lowerCAmelCase		)
          else:
               UpperCAmelCase__							: Dict 							=						F"Expected a list of numbers as input, found {type(lowerCAmelCase		).__name__}"
               raise TypeError(lowerCAmelCase		)
     else:
          raise ValueError("Missing an input"		)



def 							a__ (				lowerCAmelCase	:       list       ,    lowerCAmelCase	:       list		):






     '''simple docstring'''
     _validate_point(lowerCAmelCase		)
     _validate_point(lowerCAmelCase		)
     if len(lowerCAmelCase		) != len(lowerCAmelCase		):
          raise ValueError("Both points must be in the same n-dimensional space"		)

     return float(sum(abs(x - y		) for x, y in zip(lowerCAmelCase       ,    lowerCAmelCase		)		)		)


if __name__ == "__main__":
    import doctest

    doctest.testmod()



