import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case__				= logging.get_logger(__name__)

snake_case__				= '▁'

snake_case__				= {'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

snake_case__				= {
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

snake_case__				= {
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

snake_case__				= {
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

snake_case__				= {
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}



class  UpperCamelCase				( __lowercase ):

     '''simple docstring'''

     A_										=  ['input_ids']

     A_										=  VOCAB_FILES_NAMES
     A_										=  PRETRAINED_INIT_CONFIGURATION
     A_										=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     A_										=  PRETRAINED_VOCAB_FILES_MAP
     A_										=  RESOURCE_FILES_NAMES



     def __init__(					self				,				A_				,				A_=None				,				A_=False				,				A_="utf8"				,				A_="[UNK]"				,				A_="[SEP]"				,				A_="[PAD]"				,				A_="[CLS]"				,				A_="[MASK]"				,				A_ = None				,				**A_				,				) ->				None:






           """simple docstring"""
           _lowerCamelCase			    =				{} if sp_model_kwargs is None else sp_model_kwargs
           super().__init__(
               do_lower_case=A_				,				unk_token=A_				,				sep_token=A_				,				pad_token=A_				,				cls_token=A_				,				mask_token=A_				,				vocab_file=A_				,				encoding=A_				,				sp_model_kwargs=self.sp_model_kwargs				,				**A_				,				)
           _lowerCamelCase			    =				do_lower_case
           _lowerCamelCase			    =				sentencepiece_model_ckpt
           _lowerCamelCase			    =				spm.SentencePieceProcessor(**self.sp_model_kwargs						)
           self.sp_model.Load(A_						)

           # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
           if vocab_file is not None:
                 _lowerCamelCase			    =				self.load_vocab(filepath=A_						)
           else:
                 _lowerCamelCase			    =				{self.sp_model.id_to_piece(A_						): id for id in range(self.sp_model.get_piece_size()						)}
           _lowerCamelCase			    =				{v: k for k, v in self.vocab.items()}



     def 		UpperCamelCase_						(					self				,				A_						) ->				str:






           """simple docstring"""
           if text is None:
                 return None

           _lowerCamelCase			    =				self.tokenize(A_						)
           _lowerCamelCase    ,					_lowerCamelCase			    =				'''''', []

           for i, ch in enumerate(A_						):
                 if ch in self.SP_CHAR_MAPPING:
                       _lowerCamelCase			    =				self.SP_CHAR_MAPPING.get(A_						)
                 else:
                       _lowerCamelCase			    =				unicodedata.normalize('''NFKC'''				,				A_						)
                 if self.is_whitespace(A_						):
                       continue
                 normalized_text += ch
                 char_mapping.extend([i] * len(A_						)						)

           _lowerCamelCase    ,					_lowerCamelCase    ,					_lowerCamelCase			    =				normalized_text, [], 0

           if self.do_lower_case:
                 _lowerCamelCase			    =				text.lower()

           for token in split_tokens:
                 if token[:1] == "▁":
                       _lowerCamelCase			    =				token[1:]
                 _lowerCamelCase			    =				text[offset:].index(A_						) + offset
                 _lowerCamelCase			    =				start + len(A_						)

                 token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)						)
                 _lowerCamelCase			    =				end
           return token_mapping



     @property
     def 		UpperCamelCase_						(					self						) ->				Tuple:






           """simple docstring"""
           return len(self.vocab						)



     def 		UpperCamelCase_						(					self						) ->				Dict:






           """simple docstring"""
           return dict(self.vocab				,				**self.added_tokens_encoder						)



     def __getstate__(					self						) ->				Union[str, Any]:






           """simple docstring"""
           _lowerCamelCase			    =				self.__dict__.copy()
           _lowerCamelCase			    =				None
           return state



     def __setstate__(					self				,				A_						) ->				str:






           """simple docstring"""
           _lowerCamelCase			    =				d

           # for backward compatibility
           if not hasattr(self				,				'''sp_model_kwargs'''						):
                 _lowerCamelCase			    =				{}

           _lowerCamelCase			    =				spm.SentencePieceProcessor(**self.sp_model_kwargs						)
           self.sp_model.Load(self.sentencepiece_model_ckpt						)



     def 		UpperCamelCase_						(					self				,				A_						) ->				int:






           """simple docstring"""
           return "".join((self.SP_CHAR_MAPPING.get(A_				,				A_						) for c in text)						)



     def 		UpperCamelCase_						(					self				,				A_				,				A_=False				,				A_=64				,				A_=0.1						) ->				List[str]:






           """simple docstring"""

           if self.sp_model_kwargs.get('''enable_sampling'''						) is True:
                 _lowerCamelCase			    =				True
           if self.sp_model_kwargs.get('''alpha'''						) is not None:
                 _lowerCamelCase			    =				self.sp_model_kwargs.get('''alpha'''						)
           if self.sp_model_kwargs.get('''nbest_size'''						) is not None:
                 _lowerCamelCase			    =				self.sp_model_kwargs.get('''nbest_size'''						)

           if not enable_sampling:
                 _lowerCamelCase			    =				self.sp_model.EncodeAsPieces(A_						)
           else:
                 _lowerCamelCase			    =				self.sp_model.SampleEncodeAsPieces(A_				,				A_				,				A_						)
           _lowerCamelCase			    =				[]
           for pi, piece in enumerate(A_						):
                 if piece == SPIECE_UNDERLINE:
                       if not pieces[pi + 1].startswith(A_						) and pi != 0:
                             new_pieces.append(A_						)
                             continue
                       else:
                             continue
                 _lowerCamelCase			    =				0
                 for i, chunk in enumerate(A_						):
                       if chunk == SPIECE_UNDERLINE:
                             continue
                       if self.is_ch_char(A_						) or self.is_punct(A_						):
                             if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                   new_pieces.append(piece[lst_i:i]						)
                             new_pieces.append(A_						)
                             _lowerCamelCase			    =				i + 1
                       elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                             if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                   new_pieces.append(piece[lst_i:i]						)
                             _lowerCamelCase			    =				i
                       elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                             if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                   new_pieces.append(piece[lst_i:i]						)
                             _lowerCamelCase			    =				i
                 if len(A_						) > lst_i:
                       new_pieces.append(piece[lst_i:]						)
           return new_pieces



     def 		UpperCamelCase_						(					self				,				A_						) ->				List[str]:






           """simple docstring"""
           _lowerCamelCase			    =				''''''.join(A_						).replace(A_				,				''' '''						).strip()
           return out_string



     def 		UpperCamelCase_						(					self				,				A_						) ->				Any:






           """simple docstring"""
           _lowerCamelCase			    =				self.convert_ids_to_tokens(A_						)
           _lowerCamelCase			    =				''''''.join(A_						).replace(A_				,				''' '''						).strip()
           return out_string



     def 		UpperCamelCase_						(					self				,				A_						) ->				List[str]:






           """simple docstring"""
           return self.vocab.get(A_				,				self.vocab.get(self.unk_token						)						)



     def 		UpperCamelCase_						(					self				,				A_						) ->				Union[str, Any]:






           """simple docstring"""
           return self.reverse_vocab.get(A_				,				self.unk_token						)



     def 		UpperCamelCase_						(					self				,				A_				,				A_=None						) ->				List[str]:






           """simple docstring"""
           if token_ids_a is None:
                 return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           _lowerCamelCase			    =				[self.cls_token_id]
           _lowerCamelCase			    =				[self.sep_token_id]
           return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



     def 		UpperCamelCase_						(					self				,				A_				,				A_=None						) ->				List[Any]:






           """simple docstring"""
           if offset_mapping_a is None:
                 return [(0, 0)] + offset_mapping_a + [(0, 0)]

           return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



     def 		UpperCamelCase_						(					self				,				A_				,				A_=None				,				A_=False						) ->				List[str]:






           """simple docstring"""

           if already_has_special_tokens:
                 if token_ids_a is not None:
                       raise ValueError(
                           '''You should not supply a second sequence if the provided sequence of '''
                           '''ids is already formatted with special tokens for the model.'''						)
                 return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

           if token_ids_a is not None:
                 return [1] + ([0] * len(A_						)) + [1, 1] + ([0] * len(A_						)) + [1]
           return [1] + ([0] * len(A_						)) + [1]



     def 		UpperCamelCase_						(					self				,				A_				,				A_ = None						) ->				List[int]:






           """simple docstring"""
           # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
           if token_ids_a is None:
                 # [CLS] X [SEP]
                 return (len(A_						) + 2) * [0]

           # [CLS] A [SEP] [SEP] B [SEP]
           return [0] * (len(A_						) + 1) + [1] * (len(A_						) + 3)



     def 		UpperCamelCase_						(					self				,				A_						) ->				Dict:






           """simple docstring"""
           if "\u4e00" <= char <= "\u9fff":
                 return True
           return False



     def 		UpperCamelCase_						(					self				,				A_						) ->				Optional[Any]:






           """simple docstring"""
           if ("a" <= char <= "z") or ("A" <= char <= "Z"):
                 return True
           return False



     def 		UpperCamelCase_						(					self				,				A_						) ->				Union[str, Any]:






           """simple docstring"""
           if char in ",;:.?!~，；：。？！《》【】":
                 return True
           return False



     def 		UpperCamelCase_						(					self				,				A_						) ->				int:






           """simple docstring"""
           if char == " " or char == "\t" or char == "\n" or char == "\r":
                 return True
           if len(A_						) == 1:
                 _lowerCamelCase			    =				unicodedata.category(A_						)
                 if cat == "Zs":
                       return True
           return False



     def 		UpperCamelCase_						(					self				,				A_						) ->				Tuple:






           """simple docstring"""
           _lowerCamelCase			    =				{}
           with io.open(A_				,				'''r'''				,				encoding='''utf-8'''						) as f:
                 for index, line in enumerate(A_						):
                       _lowerCamelCase			    =				line.rstrip('''\n'''						)
                       _lowerCamelCase			    =				int(A_						)

           return token_to_idx





     def 		UpperCamelCase_						(					self				,				A_				,				A_ = None						) ->				Tuple[str]:






           """simple docstring"""
           _lowerCamelCase			    =				0
           if os.path.isdir(A_						):
                 _lowerCamelCase			    =				os.path.join(
                     A_				,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
           else:
                 _lowerCamelCase			    =				(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
           with open(A_				,				'''w'''				,				encoding='''utf-8'''						) as writer:
                 for token, token_index in sorted(self.vocab.items()				,				key=lambda A_						: kv[1]						):
                       if index != token_index:
                             logger.warning(
                                 F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                                 ''' Please check that the vocabulary is not corrupted!'''						)
                             _lowerCamelCase			    =				token_index
                       writer.write(token + '''\n'''						)
                       index += 1

           _lowerCamelCase			    =				os.path.join(A_				,				'''sentencepiece.bpe.model'''						)
           with open(A_				,				'''wb'''						) as fi:
                 _lowerCamelCase			    =				self.sp_model.serialized_model_proto()
                 fi.write(A_						)

           return (vocab_file,)


def       __magic_name__(  __UpperCAmelCase ,						__UpperCAmelCase		)						->    bool:



      '''simple docstring'''

      _lowerCamelCase			    =				len(__UpperCAmelCase		)
      _lowerCamelCase			    =				[[False] * (required_sum + 1) for _ in range(arr_len + 1		)]

      # for each arr value, a sum of zero(0) can be formed by not taking any element
      # hence True/1
      for i in range(arr_len + 1		):
            _lowerCamelCase			    =				True

      # sum is not zero and set is empty then false
      for i in range(1 ,						required_sum + 1		):
            _lowerCamelCase			    =				False

      for i in range(1 ,						arr_len + 1		):
            for j in range(1 ,						required_sum + 1		):
                  if arr[i - 1] > j:
                        _lowerCamelCase			    =				subset[i - 1][j]
                  if arr[i - 1] <= j:
                        _lowerCamelCase			    =				subset[i - 1][j] or subset[i - 1][j - arr[i - 1]]

      return subset[arr_len][required_sum]


if __name__ == "__main__":
     import doctest

     doctest.testmod()