





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowercase_		:				Any				=						logging.get_logger(__name__)

lowercase_		:				Dict				=						{'''vocab_file''': '''spiece.model'''}

lowercase_		:				Union[str, Any]				=						{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

lowercase_		:				Tuple				=						{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
lowercase_		:				List[str]				=						0
lowercase_		:				List[str]				=						1
lowercase_		:				List[str]				=						2
lowercase_		:				Tuple				=						3
lowercase_		:				Optional[int]				=						4


class 			lowercase				(							a_     ):



      """simple docstring"""

      _UpperCamelCase  :	Any								=	VOCAB_FILES_NAMES
      _UpperCamelCase  :	Any								=	PRETRAINED_VOCAB_FILES_MAP
      _UpperCamelCase  :	Dict								=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _UpperCamelCase  :	Any								=	"left"







      def __init__(    self			:		Dict ,				lowerCamelCase_			:		List[Any] ,				lowerCamelCase_			:		Dict=False ,				lowerCamelCase_			:		int=True ,				lowerCamelCase_			:		List[Any]=False ,				lowerCamelCase_			:		List[str]="<s>" ,				lowerCamelCase_			:		List[Any]="</s>" ,				lowerCamelCase_			:		Optional[Any]="<unk>" ,				lowerCamelCase_			:		int="<sep>" ,				lowerCamelCase_			:		int="<pad>" ,				lowerCamelCase_			:		Optional[int]="<cls>" ,				lowerCamelCase_			:		int="<mask>" ,				lowerCamelCase_			:		Optional[int]=["<eop>", "<eod>"] ,				lowerCamelCase_			:		Optional[Dict[str, Any]] = None ,				**lowerCamelCase_			:		List[Any] ,				):





          '''simple docstring'''







          _snake_case :			Tuple  						=   AddedToken(UpperCamelCase_ ,				lstrip=UpperCamelCase_ ,				rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_ ,				UpperCamelCase_					) else mask_token

          _snake_case :			Any  						=   {} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              do_lower_case=UpperCamelCase_ ,				remove_space=UpperCamelCase_ ,				keep_accents=UpperCamelCase_ ,				bos_token=UpperCamelCase_ ,				eos_token=UpperCamelCase_ ,				unk_token=UpperCamelCase_ ,				sep_token=UpperCamelCase_ ,				pad_token=UpperCamelCase_ ,				cls_token=UpperCamelCase_ ,				mask_token=UpperCamelCase_ ,				additional_special_tokens=UpperCamelCase_ ,				sp_model_kwargs=self.sp_model_kwargs ,				**UpperCamelCase_ ,				)

          _snake_case :			List[Any]  						=   3

          _snake_case :			List[str]  						=   do_lower_case
          _snake_case :			List[Any]  						=   remove_space
          _snake_case :			Union[str, Any]  						=   keep_accents
          _snake_case :			Optional[Any]  						=   vocab_file

          _snake_case :			Tuple  						=   spm.SentencePieceProcessor(**self.sp_model_kwargs					)
          self.sp_model.Load(UpperCamelCase_					)







      @property
      def 		__UpperCAmelCase       (    self			:		List[Any]					):





          '''simple docstring'''







          return len(self.sp_model					)







      def 		__UpperCAmelCase       (    self			:		Union[str, Any]					):





          '''simple docstring'''







          _snake_case :			Tuple  						=   {self.convert_ids_to_tokens(UpperCamelCase_					): i for i in range(self.vocab_size					)}
          vocab.update(self.added_tokens_encoder					)
          return vocab







      def __getstate__(    self			:		List[str]					):





          '''simple docstring'''







          _snake_case :			int  						=   self.__dict__.copy()
          _snake_case :			Optional[int]  						=   None
          return state







      def __setstate__(    self			:		List[Any] ,				lowerCamelCase_			:		Tuple					):





          '''simple docstring'''







          _snake_case :			Optional[int]  						=   d

          # for backward compatibility
          if not hasattr(self ,				'sp_model_kwargs'					):
              _snake_case :			int  						=   {}

          _snake_case :			Any  						=   spm.SentencePieceProcessor(**self.sp_model_kwargs					)
          self.sp_model.Load(self.vocab_file					)







      def 		__UpperCAmelCase       (    self			:		Dict ,				lowerCamelCase_			:		Union[str, Any]					):





          '''simple docstring'''







          if self.remove_space:
              _snake_case :			Optional[int]  						=   ' '.join(inputs.strip().split()					)
          else:
              _snake_case :			List[str]  						=   inputs
          _snake_case :			int  						=   outputs.replace('``' ,				'"'					).replace('\'\'' ,				'"'					)

          if not self.keep_accents:
              _snake_case :			Tuple  						=   unicodedata.normalize('NFKD' ,				UpperCamelCase_					)
              _snake_case :			Any  						=   ''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase_					)]					)
          if self.do_lower_case:
              _snake_case :			int  						=   outputs.lower()

          return outputs







      def 		__UpperCAmelCase       (    self			:		Optional[Any] ,				lowerCamelCase_			:		str					):





          '''simple docstring'''







          _snake_case :			Tuple  						=   self.preprocess_text(UpperCamelCase_					)
          _snake_case :			Dict  						=   self.sp_model.encode(UpperCamelCase_ ,				out_type=UpperCamelCase_					)
          _snake_case :			List[Any]  						=   []
          for piece in pieces:
              if len(UpperCamelCase_					) > 1 and piece[-1] == str(','					) and piece[-2].isdigit():
                  _snake_case :			int  						=   self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase_ ,				''					)					)
                  if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                      if len(cur_pieces[0]					) == 1:
                          _snake_case :			Optional[int]  						=   cur_pieces[1:]
                      else:
                          _snake_case :			Tuple  						=   cur_pieces[0][1:]
                  cur_pieces.append(piece[-1]					)
                  new_pieces.extend(UpperCamelCase_					)
              else:
                  new_pieces.append(UpperCamelCase_					)

          return new_pieces







      def 		__UpperCAmelCase       (    self			:		Union[str, Any] ,				lowerCamelCase_			:		Optional[Any]					):





          '''simple docstring'''







          return self.sp_model.PieceToId(UpperCamelCase_					)







      def 		__UpperCAmelCase       (    self			:		List[Any] ,				lowerCamelCase_			:		Optional[Any]					):





          '''simple docstring'''







          return self.sp_model.IdToPiece(UpperCamelCase_					)







      def 		__UpperCAmelCase       (    self			:		int ,				lowerCamelCase_			:		Any					):





          '''simple docstring'''







          _snake_case :			List[Any]  						=   ''.join(UpperCamelCase_					).replace(UpperCamelCase_ ,				' '					).strip()
          return out_string







      def 		__UpperCAmelCase       (    self			:		Any ,				lowerCamelCase_			:		List[int] ,				lowerCamelCase_			:		bool = False ,				lowerCamelCase_			:		bool = None ,				lowerCamelCase_			:		bool = True ,				**lowerCamelCase_			:		Union[str, Any] ,				):





          '''simple docstring'''







          _snake_case :			Union[str, Any]  						=   kwargs.pop('use_source_tokenizer' ,				UpperCamelCase_					)

          _snake_case :			int  						=   self.convert_ids_to_tokens(UpperCamelCase_ ,				skip_special_tokens=UpperCamelCase_					)

          # To avoid mixing byte-level and unicode for byte-level BPT
          # we need to build string separately for added tokens and byte-level tokens
          # cf. https://github.com/huggingface/transformers/issues/1133
          _snake_case :			List[Any]  						=   []
          _snake_case :			int  						=   []
          for token in filtered_tokens:
              if skip_special_tokens and token in self.all_special_ids:
                  continue
              if token in self.added_tokens_encoder:
                  if current_sub_text:
                      sub_texts.append(self.convert_tokens_to_string(UpperCamelCase_					)					)
                      _snake_case :			Optional[int]  						=   []
                  sub_texts.append(UpperCamelCase_					)
              else:
                  current_sub_text.append(UpperCamelCase_					)
          if current_sub_text:
              sub_texts.append(self.convert_tokens_to_string(UpperCamelCase_					)					)

          # Mimic the behavior of the Rust tokenizer:
          # By default, there are no spaces between special tokens
          _snake_case :			Tuple  						=   ''.join(UpperCamelCase_					)

          _snake_case :			int  						=   (
              clean_up_tokenization_spaces
              if clean_up_tokenization_spaces is not None
              else self.clean_up_tokenization_spaces
          )
          if clean_up_tokenization_spaces:
              _snake_case :			List[str]  						=   self.clean_up_tokenization(UpperCamelCase_					)
              return clean_text
          else:
              return text







      def 		__UpperCAmelCase       (    self			:		List[str] ,				lowerCamelCase_			:		List[int] ,				lowerCamelCase_			:		Optional[List[int]] = None					):





          '''simple docstring'''







          _snake_case :			Any  						=   [self.sep_token_id]
          _snake_case :			List[Any]  						=   [self.cls_token_id]
          if token_ids_a is None:
              return token_ids_a + sep + cls
          return token_ids_a + sep + token_ids_a + sep + cls







      def 		__UpperCAmelCase       (    self			:		Optional[Any] ,				lowerCamelCase_			:		List[int] ,				lowerCamelCase_			:		Optional[List[int]] = None ,				lowerCamelCase_			:		bool = False					):





          '''simple docstring'''







          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=UpperCamelCase_ ,				token_ids_a=UpperCamelCase_ ,				already_has_special_tokens=UpperCamelCase_					)

          if token_ids_a is not None:
              return ([0] * len(UpperCamelCase_					)) + [1] + ([0] * len(UpperCamelCase_					)) + [1, 1]
          return ([0] * len(UpperCamelCase_					)) + [1, 1]







      def 		__UpperCAmelCase       (    self			:		List[str] ,				lowerCamelCase_			:		List[int] ,				lowerCamelCase_			:		Optional[List[int]] = None					):





          '''simple docstring'''







          _snake_case :			Union[str, Any]  						=   [self.sep_token_id]
          _snake_case :			Union[str, Any]  						=   [2]

          if token_ids_a is None:
              return len(token_ids_a + sep					) * [0] + cls_segment_id
          return len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1] + cls_segment_id







      def 		__UpperCAmelCase       (    self			:		Union[str, Any] ,				lowerCamelCase_			:		str ,				lowerCamelCase_			:		Optional[str] = None					):





          '''simple docstring'''







          if not os.path.isdir(UpperCamelCase_					):
              logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
              return
          _snake_case :			List[Any]  						=   os.path.join(
              UpperCamelCase_ ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

          if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCamelCase_					) and os.path.isfile(self.vocab_file					):
              copyfile(self.vocab_file ,				UpperCamelCase_					)
          elif not os.path.isfile(self.vocab_file					):
              with open(UpperCamelCase_ ,				'wb'					) as fi:
                  _snake_case :			List[Any]  						=   self.sp_model.serialized_model_proto()
                  fi.write(UpperCamelCase_					)

          return (out_vocab_file,)













from math import factorial

def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				if successes > trials:
								raise ValueError('successes must be lower or equal to trials' )
				if trials < 0 or successes < 0:
								raise ValueError('the function is defined for non-negative integers' )
				if not isinstance(__lowerCAmelCase	,			__lowerCAmelCase ) or not isinstance(__lowerCAmelCase	,			__lowerCAmelCase ):
								raise ValueError('the function is defined for non-negative integers' )
				if not 0 < prob < 1:
								raise ValueError('prob has to be in range of 1 - 0' )
				_snake_case :			Optional[int]  						=   (prob**successes) * ((1 - prob) ** (trials - successes))
				# Calculate the binomial coefficient: n! / k!(n-k)!
				_snake_case :			List[Any]  						=   float(factorial(__lowerCAmelCase ) )
				coefficient /= factorial(__lowerCAmelCase ) * factorial(trials - successes )
				return probability * coefficient


if __name__ == "__main__":
		from doctest import testmod

		testmod()
		print('''Probability of 2 successes out of 4 trails''')
		print('''with probability of 0.75 is:''', end=''' ''')
		print(binomial_distribution(2, 4, 0.75))






