



"""simple docstring"""






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case							:     Dict   =					logging.get_logger(__name__)

__snake_case							:     int   =					'''▁'''

__snake_case							:     Dict   =					{'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

__snake_case							:     Tuple   =					{
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

__snake_case							:     Tuple   =					{
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

__snake_case							:     Any   =					{
    '''ernie-m-base''': 514,
    '''ernie-m-large''': 514,
}

__snake_case							:     Dict   =					{
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}



class     A__     (						a__							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE					   =			["input_ids"]

	SCREAMING_SNAKE_CASE					   =			VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE					   =			PRETRAINED_INIT_CONFIGURATION
	SCREAMING_SNAKE_CASE					   =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE					   =			PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE					   =			RESOURCE_FILES_NAMES

	def __init__(  self:	List[str] ,    _SCREAMING_SNAKE_CASE:	Optional[int] ,    _SCREAMING_SNAKE_CASE:	Optional[Any]=None ,    _SCREAMING_SNAKE_CASE:	int=False ,    _SCREAMING_SNAKE_CASE:	List[str]="utf8" ,    _SCREAMING_SNAKE_CASE:	Optional[Any]="[UNK]" ,    _SCREAMING_SNAKE_CASE:	Tuple="[SEP]" ,    _SCREAMING_SNAKE_CASE:	Optional[Any]="[PAD]" ,    _SCREAMING_SNAKE_CASE:	Optional[int]="[CLS]" ,    _SCREAMING_SNAKE_CASE:	Union[str, Any]="[MASK]" ,    _SCREAMING_SNAKE_CASE:	Any = None ,    **_SCREAMING_SNAKE_CASE:	Optional[Any] ,    )       -> None:

				"""simple docstring"""

				__lowerCAmelCase   :						Optional[int]				= {} if sp_model_kwargs is None else sp_model_kwargs
				super().__init__(
				    do_lower_case=lowerCAmelCase__ ,    unk_token=lowerCAmelCase__ ,    sep_token=lowerCAmelCase__ ,    pad_token=lowerCAmelCase__ ,    cls_token=lowerCAmelCase__ ,    mask_token=lowerCAmelCase__ ,    vocab_file=lowerCAmelCase__ ,    encoding=lowerCAmelCase__ ,    sp_model_kwargs=self.sp_model_kwargs ,    **lowerCAmelCase__ ,    )
				__lowerCAmelCase   :						List[str]				= do_lower_case
				__lowerCAmelCase   :						Any				= sentencepiece_model_ckpt
				__lowerCAmelCase   :						Optional[Any]				= spm.SentencePieceProcessor(**self.sp_model_kwargs)
				self.sp_model.Load(lowerCAmelCase__)

				# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
				if vocab_file is not None:
							__lowerCAmelCase   :						Any				= self.load_vocab(filepath=lowerCAmelCase__)
				else:
							__lowerCAmelCase   :						Dict				= {self.sp_model.id_to_piece(lowerCAmelCase__): id for id in range(self.sp_model.get_piece_size())}
				__lowerCAmelCase   :						Any				= {v: k for k, v in self.vocab.items()}

	def 							_SCREAMING_SNAKE_CASE     (  self:	int ,    _SCREAMING_SNAKE_CASE:	int)       -> Any:

				"""simple docstring"""

				if text is None:
							return None

				__lowerCAmelCase   :						List[Any]				= self.tokenize(lowerCAmelCase__)
				__lowerCAmelCase   :						Any				= "", []

				for i, ch in enumerate(lowerCAmelCase__):
							if ch in self.SP_CHAR_MAPPING:
										__lowerCAmelCase   :						Any				= self.SP_CHAR_MAPPING.get(lowerCAmelCase__)
							else:
										__lowerCAmelCase   :						Optional[Any]				= unicodedata.normalize("NFKC" ,    lowerCAmelCase__)
							if self.is_whitespace(lowerCAmelCase__):
										continue
							normalized_text += ch
							char_mapping.extend([i] * len(lowerCAmelCase__))

				__lowerCAmelCase   :						Union[str, Any]				= normalized_text, [], 0

				if self.do_lower_case:
							__lowerCAmelCase   :						List[Any]				= text.lower()

				for token in split_tokens:
							if token[:1] == "▁":
										__lowerCAmelCase   :						Tuple				= token[1:]
							__lowerCAmelCase   :						Any				= text[offset:].index(lowerCAmelCase__) + offset
							__lowerCAmelCase   :						Union[str, Any]				= start + len(lowerCAmelCase__)

							token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1))
							__lowerCAmelCase   :						Any				= end
				return token_mapping

	@property
	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Dict:

				"""simple docstring"""

				return len(self.vocab)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> List[Any]:

				"""simple docstring"""

				return dict(self.vocab ,    **self.added_tokens_encoder)

	def __getstate__(  self:	Union[str, Any])       -> Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						Optional[int]				= self.__dict__.copy()
				__lowerCAmelCase   :						Any				= None
				return state

	def __setstate__(  self:	Optional[Any] ,    _SCREAMING_SNAKE_CASE:	Dict)       -> Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= d

				# for backward compatibility
				if not hasattr(self ,    "sp_model_kwargs"):
							__lowerCAmelCase   :						int				= {}

				__lowerCAmelCase   :						Any				= spm.SentencePieceProcessor(**self.sp_model_kwargs)
				self.sp_model.Load(self.sentencepiece_model_ckpt)

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any] ,    _SCREAMING_SNAKE_CASE:	List[Any])       -> List[str]:

				"""simple docstring"""

				return "".join((self.SP_CHAR_MAPPING.get(lowerCAmelCase__ ,    lowerCAmelCase__) for c in text))

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str] ,    _SCREAMING_SNAKE_CASE:	Union[str, Any] ,    _SCREAMING_SNAKE_CASE:	List[str]=False ,    _SCREAMING_SNAKE_CASE:	Optional[Any]=64 ,    _SCREAMING_SNAKE_CASE:	Tuple=0.1)       -> List[Any]:

				"""simple docstring"""

				if self.sp_model_kwargs.get("enable_sampling") is True:
							__lowerCAmelCase   :						int				= True
				if self.sp_model_kwargs.get("alpha") is not None:
							__lowerCAmelCase   :						Any				= self.sp_model_kwargs.get("alpha")
				if self.sp_model_kwargs.get("nbest_size") is not None:
							__lowerCAmelCase   :						str				= self.sp_model_kwargs.get("nbest_size")

				if not enable_sampling:
							__lowerCAmelCase   :						int				= self.sp_model.EncodeAsPieces(lowerCAmelCase__)
				else:
							__lowerCAmelCase   :						List[str]				= self.sp_model.SampleEncodeAsPieces(lowerCAmelCase__ ,    lowerCAmelCase__ ,    lowerCAmelCase__)
				__lowerCAmelCase   :						Optional[Any]				= []
				for pi, piece in enumerate(lowerCAmelCase__):
							if piece == SPIECE_UNDERLINE:
										if not pieces[pi + 1].startswith(lowerCAmelCase__) and pi != 0:
													new_pieces.append(lowerCAmelCase__)
													continue
										else:
													continue
							__lowerCAmelCase   :						Union[str, Any]				= 0
							for i, chunk in enumerate(lowerCAmelCase__):
										if chunk == SPIECE_UNDERLINE:
													continue
										if self.is_ch_char(lowerCAmelCase__) or self.is_punct(lowerCAmelCase__):
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i])
													new_pieces.append(lowerCAmelCase__)
													__lowerCAmelCase   :						Tuple				= i + 1
										elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i])
													__lowerCAmelCase   :						Dict				= i
										elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i])
													__lowerCAmelCase   :						Optional[Any]				= i
							if len(lowerCAmelCase__) > lst_i:
										new_pieces.append(piece[lst_i:])
				return new_pieces

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[Any] ,    _SCREAMING_SNAKE_CASE:	List[Any])       -> Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						Any				= "".join(lowerCAmelCase__).replace(lowerCAmelCase__ ,    " ").strip()
				return out_string

	def 							_SCREAMING_SNAKE_CASE     (  self:	Tuple ,    _SCREAMING_SNAKE_CASE:	Optional[int])       -> Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase   :						Optional[Any]				= self.convert_ids_to_tokens(lowerCAmelCase__)
				__lowerCAmelCase   :						Optional[Any]				= "".join(lowerCAmelCase__).replace(lowerCAmelCase__ ,    " ").strip()
				return out_string

	def 							_SCREAMING_SNAKE_CASE     (  self:	int ,    _SCREAMING_SNAKE_CASE:	Any)       -> str:

				"""simple docstring"""

				return self.vocab.get(lowerCAmelCase__ ,    self.vocab.get(self.unk_token))

	def 							_SCREAMING_SNAKE_CASE     (  self:	int ,    _SCREAMING_SNAKE_CASE:	Optional[Any])       -> str:

				"""simple docstring"""

				return self.reverse_vocab.get(lowerCAmelCase__ ,    self.unk_token)

	def 							_SCREAMING_SNAKE_CASE     (  self:	str ,    _SCREAMING_SNAKE_CASE:	Optional[int] ,    _SCREAMING_SNAKE_CASE:	Union[str, Any]=None)       -> Dict:

				"""simple docstring"""

				if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				__lowerCAmelCase   :						Tuple				= [self.cls_token_id]
				__lowerCAmelCase   :						Any				= [self.sep_token_id]
				return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any] ,    _SCREAMING_SNAKE_CASE:	Dict ,    _SCREAMING_SNAKE_CASE:	Optional[Any]=None)       -> Any:

				"""simple docstring"""

				if offset_mapping_a is None:
							return [(0, 0)] + offset_mapping_a + [(0, 0)]

				return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[int] ,    _SCREAMING_SNAKE_CASE:	Optional[int] ,    _SCREAMING_SNAKE_CASE:	List[str]=None ,    _SCREAMING_SNAKE_CASE:	List[Any]=False)       -> Tuple:

				"""simple docstring"""

				if already_has_special_tokens:
							if token_ids_a is not None:
										raise ValueError(
										    "You should not supply a second sequence if the provided sequence of "
										    "ids is already formatted with special tokens for the model.")
							return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

				if token_ids_a is not None:
							return [1] + ([0] * len(lowerCAmelCase__)) + [1, 1] + ([0] * len(lowerCAmelCase__)) + [1]
				return [1] + ([0] * len(lowerCAmelCase__)) + [1]

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any] ,    _SCREAMING_SNAKE_CASE:	Dict ,    _SCREAMING_SNAKE_CASE:	int = None)       -> List[int]:

				"""simple docstring"""

				if token_ids_a is None:
							# [CLS] X [SEP]
							return (len(lowerCAmelCase__) + 2) * [0]

				# [CLS] A [SEP] [SEP] B [SEP]
				return [0] * (len(lowerCAmelCase__) + 1) + [1] * (len(lowerCAmelCase__) + 3)

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any] ,    _SCREAMING_SNAKE_CASE:	Optional[int])       -> List[str]:

				"""simple docstring"""

				if "\u4e00" <= char <= "\u9fff":
							return True
				return False

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any ,    _SCREAMING_SNAKE_CASE:	Tuple)       -> str:

				"""simple docstring"""

				if ("a" <= char <= "z") or ("A" <= char <= "Z"):
							return True
				return False

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any] ,    _SCREAMING_SNAKE_CASE:	str)       -> Any:

				"""simple docstring"""

				if char in ",;:.?!~，；：。？！《》【】":
							return True
				return False

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any ,    _SCREAMING_SNAKE_CASE:	str)       -> Dict:

				"""simple docstring"""

				if char == " " or char == "\t" or char == "\n" or char == "\r":
							return True
				if len(lowerCAmelCase__) == 1:
							__lowerCAmelCase   :						List[str]				= unicodedata.category(lowerCAmelCase__)
							if cat == "Zs":
										return True
				return False

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any] ,    _SCREAMING_SNAKE_CASE:	Dict)       -> Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase   :						List[Any]				= {}
				with io.open(lowerCAmelCase__ ,    "r" ,    encoding="utf-8") as f:
							for index, line in enumerate(lowerCAmelCase__):
										__lowerCAmelCase   :						List[Any]				= line.rstrip("\n")
										__lowerCAmelCase   :						List[Any]				= int(lowerCAmelCase__)

				return token_to_idx







	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str] ,    _SCREAMING_SNAKE_CASE:	Union[str, Any] ,    _SCREAMING_SNAKE_CASE:	Optional[Any] = None)       -> Tuple[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						List[str]				= 0
				if os.path.isdir(lowerCAmelCase__):
							__lowerCAmelCase   :						Dict				= os.path.join(
							    lowerCAmelCase__ ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
				else:
							__lowerCAmelCase   :						Optional[int]				= (filename_prefix + "-" if filename_prefix else "") + save_directory
				with open(lowerCAmelCase__ ,    "w" ,    encoding="utf-8") as writer:
							for token, token_index in sorted(self.vocab.items() ,    key=lambda _SCREAMING_SNAKE_CASE: kv[1]):
										if index != token_index:
													logger.warning(
													    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
													    " Please check that the vocabulary is not corrupted!")
													__lowerCAmelCase   :						str				= token_index
										writer.write(token + "\n")
										index += 1

				__lowerCAmelCase   :						Any				= os.path.join(lowerCAmelCase__ ,    "sentencepiece.bpe.model")
				with open(lowerCAmelCase__ ,    "wb") as fi:
							__lowerCAmelCase   :						Union[str, Any]				= self.sp_model.serialized_model_proto()
							fi.write(lowerCAmelCase__)

				return (vocab_file,)









"""simple docstring"""






import unittest

from knapsack import knapsack as k



class     A__     (						unittest.TestCase							):




	'''simple docstring'''





	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[int])       -> List[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						List[Any]				= 0
				__lowerCAmelCase   :						str				= [0]
				__lowerCAmelCase   :						Union[str, Any]				= [0]
				__lowerCAmelCase   :						List[Any]				= len(_SCREAMING_SNAKE_CASE)
				self.assertEqual(k.knapsack(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE) ,    0)

				__lowerCAmelCase   :						Dict				= [60]
				__lowerCAmelCase   :						Optional[int]				= [10]
				__lowerCAmelCase   :						Tuple				= len(_SCREAMING_SNAKE_CASE)
				self.assertEqual(k.knapsack(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE) ,    0)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> int:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= 3
				__lowerCAmelCase   :						Dict				= [1, 2, 3]
				__lowerCAmelCase   :						str				= [3, 2, 1]
				__lowerCAmelCase   :						Optional[int]				= len(_SCREAMING_SNAKE_CASE)
				self.assertEqual(k.knapsack(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE) ,    5)







	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[int])       -> Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						List[str]				= 50
				__lowerCAmelCase   :						Dict				= [60, 100, 120]
				__lowerCAmelCase   :						List[Any]				= [10, 20, 30]
				__lowerCAmelCase   :						int				= len(_SCREAMING_SNAKE_CASE)
				self.assertEqual(k.knapsack(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE) ,    220)


if __name__ == "__main__":
	unittest.main()