


"""simple docstring"""


import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case          =							logging.get_logger(__name__)

_snake_case          =							"▁"

_snake_case          =							{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

_snake_case          =							{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

_snake_case          =							{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

_snake_case          =							{
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

_snake_case          =							{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}



class   _a      (      SCREAMING_SNAKE_CASE_ ):
    a_    :				List[str]             =    ["input_ids"]

    a_    :				Tuple             =    VOCAB_FILES_NAMES
    a_    :				List[Any]             =    PRETRAINED_INIT_CONFIGURATION
    a_    :				int             =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a_    :				List[Any]             =    PRETRAINED_VOCAB_FILES_MAP
    a_    :				Optional[int]             =    RESOURCE_FILES_NAMES
    def __init__( self							:					Any							,					SCREAMING_SNAKE_CASE__							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					int=None							,					SCREAMING_SNAKE_CASE__							:					List[str]=False							,					SCREAMING_SNAKE_CASE__							:					Tuple="utf8"							,					SCREAMING_SNAKE_CASE__							:					Optional[int]="[UNK]"							,					SCREAMING_SNAKE_CASE__							:					str="[SEP]"							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]="[PAD]"							,					SCREAMING_SNAKE_CASE__							:					str="[CLS]"							,					SCREAMING_SNAKE_CASE__							:					Optional[int]="[MASK]"							,					SCREAMING_SNAKE_CASE__							:					Optional[Dict[str, Any]] = None							,					**SCREAMING_SNAKE_CASE__							:					Any							,					):
          # Mask token behave like a normal word, i.e. include the space before it and
          # is included in the raw text, there should be a match in a non-normalized sentence.

          lowerCamelCase__           =		{} if sp_model_kwargs is None else sp_model_kwargs
          super().__init__(
              do_lower_case=SCREAMING_SNAKE_CASE__							,					unk_token=SCREAMING_SNAKE_CASE__							,					sep_token=SCREAMING_SNAKE_CASE__							,					pad_token=SCREAMING_SNAKE_CASE__							,					cls_token=SCREAMING_SNAKE_CASE__							,					mask_token=SCREAMING_SNAKE_CASE__							,					vocab_file=SCREAMING_SNAKE_CASE__							,					encoding=SCREAMING_SNAKE_CASE__							,					sp_model_kwargs=self.sp_model_kwargs							,					**SCREAMING_SNAKE_CASE__							,					)
          lowerCamelCase__           =		do_lower_case
          lowerCamelCase__           =		sentencepiece_model_ckpt
          lowerCamelCase__           =		spm.SentencePieceProcessor(**self.sp_model_kwargs				)
          self.sp_model.Load(SCREAMING_SNAKE_CASE__				)

          # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
          if vocab_file is not None:
                lowerCamelCase__           =		self.load_vocab(filepath=SCREAMING_SNAKE_CASE__				)
          else:
                lowerCamelCase__           =		{self.sp_model.id_to_piece(SCREAMING_SNAKE_CASE__				): id for id in range(self.sp_model.get_piece_size()				)}
          lowerCamelCase__           =		{v: k for k, v in self.vocab.items()}
    def      _UpperCamelCase       ( self							:					List[Any]							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]				):
          if text is None:
                return None

          lowerCamelCase__           =		self.tokenize(SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__    ,  lowerCamelCase__           =		'', []

          for i, ch in enumerate(SCREAMING_SNAKE_CASE__				):
                if ch in self.SP_CHAR_MAPPING:
                      lowerCamelCase__           =		self.SP_CHAR_MAPPING.get(SCREAMING_SNAKE_CASE__				)
                else:
                      lowerCamelCase__           =		unicodedata.normalize('NFKC'							,					SCREAMING_SNAKE_CASE__				)
                if self.is_whitespace(SCREAMING_SNAKE_CASE__				):
                      continue
                normalized_text += ch
                char_mapping.extend([i] * len(SCREAMING_SNAKE_CASE__				)				)

          lowerCamelCase__    ,  lowerCamelCase__    ,  lowerCamelCase__           =		normalized_text, [], 0

          if self.do_lower_case:
                lowerCamelCase__           =		text.lower()

          for token in split_tokens:
                if token[:1] == "▁":
                      lowerCamelCase__           =		token[1:]
                lowerCamelCase__           =		text[offset:].index(SCREAMING_SNAKE_CASE__				) + offset
                lowerCamelCase__           =		start + len(SCREAMING_SNAKE_CASE__				)

                token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)				)
                lowerCamelCase__           =		end
          return token_mapping
    @property
    def      _UpperCamelCase       ( self							:					List[str]				):
          return len(self.vocab				)
    def      _UpperCamelCase       ( self							:					int				):
          return dict(self.vocab							,					**self.added_tokens_encoder				)
    def __getstate__( self							:					int				):
          lowerCamelCase__           =		self.__dict__.copy()
          lowerCamelCase__           =		None
          return state
    def __setstate__( self							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					Dict				):
          lowerCamelCase__           =		d

          # for backward compatibility
          if not hasattr(self							,					'sp_model_kwargs'				):
                lowerCamelCase__           =		{}

          lowerCamelCase__           =		spm.SentencePieceProcessor(**self.sp_model_kwargs				)
          self.sp_model.Load(self.sentencepiece_model_ckpt				)
    def      _UpperCamelCase       ( self							:					int							,					SCREAMING_SNAKE_CASE__							:					int				):
          return "".join((self.SP_CHAR_MAPPING.get(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				) for c in text)				)
    def      _UpperCamelCase       ( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					int							,					SCREAMING_SNAKE_CASE__							:					List[Any]=False							,					SCREAMING_SNAKE_CASE__							:					Optional[Any]=64							,					SCREAMING_SNAKE_CASE__							:					str=0.1				):

          if self.sp_model_kwargs.get('enable_sampling'				) is True:
                lowerCamelCase__           =		True
          if self.sp_model_kwargs.get('alpha'				) is not None:
                lowerCamelCase__           =		self.sp_model_kwargs.get('alpha'				)
          if self.sp_model_kwargs.get('nbest_size'				) is not None:
                lowerCamelCase__           =		self.sp_model_kwargs.get('nbest_size'				)

          if not enable_sampling:
                lowerCamelCase__           =		self.sp_model.EncodeAsPieces(SCREAMING_SNAKE_CASE__				)
          else:
                lowerCamelCase__           =		self.sp_model.SampleEncodeAsPieces(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		[]
          for pi, piece in enumerate(SCREAMING_SNAKE_CASE__				):
                if piece == SPIECE_UNDERLINE:
                      if not pieces[pi + 1].startswith(SCREAMING_SNAKE_CASE__				) and pi != 0:
                            new_pieces.append(SCREAMING_SNAKE_CASE__				)
                            continue
                      else:
                            continue
                lowerCamelCase__           =		0
                for i, chunk in enumerate(SCREAMING_SNAKE_CASE__				):
                      if chunk == SPIECE_UNDERLINE:
                            continue
                      if self.is_ch_char(SCREAMING_SNAKE_CASE__				) or self.is_punct(SCREAMING_SNAKE_CASE__				):
                            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                  new_pieces.append(piece[lst_i:i]				)
                            new_pieces.append(SCREAMING_SNAKE_CASE__				)
                            lowerCamelCase__           =		i + 1
                      elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                  new_pieces.append(piece[lst_i:i]				)
                            lowerCamelCase__           =		i
                      elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                  new_pieces.append(piece[lst_i:i]				)
                            lowerCamelCase__           =		i
                if len(SCREAMING_SNAKE_CASE__				) > lst_i:
                      new_pieces.append(piece[lst_i:]				)
          return new_pieces
    def      _UpperCamelCase       ( self							:					List[Any]							,					SCREAMING_SNAKE_CASE__							:					int				):
          lowerCamelCase__           =		''.join(SCREAMING_SNAKE_CASE__				).replace(SCREAMING_SNAKE_CASE__							,					' '				).strip()
          return out_string
    def      _UpperCamelCase       ( self							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					Any				):
          lowerCamelCase__           =		self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		''.join(SCREAMING_SNAKE_CASE__				).replace(SCREAMING_SNAKE_CASE__							,					' '				).strip()
          return out_string
    def      _UpperCamelCase       ( self							:					Any							,					SCREAMING_SNAKE_CASE__							:					int				):
          return self.vocab.get(SCREAMING_SNAKE_CASE__							,					self.vocab.get(self.unk_token				)				)
    def      _UpperCamelCase       ( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]				):
          return self.reverse_vocab.get(SCREAMING_SNAKE_CASE__							,					self.unk_token				)
    def      _UpperCamelCase       ( self							:					Any							,					SCREAMING_SNAKE_CASE__							:					Any							,					SCREAMING_SNAKE_CASE__							:					Tuple=None				):
          if token_ids_a is None:
                return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          lowerCamelCase__           =		[self.cls_token_id]
          lowerCamelCase__           =		[self.sep_token_id]
          return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep
    def      _UpperCamelCase       ( self							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					Optional[int]=None				):
          if offset_mapping_a is None:
                return [(0, 0)] + offset_mapping_a + [(0, 0)]

          return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]
    def      _UpperCamelCase       ( self							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					List[Any]							,					SCREAMING_SNAKE_CASE__							:					List[Any]=None							,					SCREAMING_SNAKE_CASE__							:					List[Any]=False				):

          if already_has_special_tokens:
                if token_ids_a is not None:
                      raise ValueError(
                          'You should not supply a second sequence if the provided sequence of '
                          'ids is already formatted with special tokens for the model.'				)
                return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

          if token_ids_a is not None:
                return [1] + ([0] * len(SCREAMING_SNAKE_CASE__				)) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE__				)) + [1]
          return [1] + ([0] * len(SCREAMING_SNAKE_CASE__				)) + [1]
    def      _UpperCamelCase       ( self							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					List[int]							,					SCREAMING_SNAKE_CASE__							:					Optional[List[int]] = None				):
          # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
          if token_ids_a is None:
                # [CLS] X [SEP]
                return (len(SCREAMING_SNAKE_CASE__				) + 2) * [0]

          # [CLS] A [SEP] [SEP] B [SEP]
          return [0] * (len(SCREAMING_SNAKE_CASE__				) + 1) + [1] * (len(SCREAMING_SNAKE_CASE__				) + 3)
    def      _UpperCamelCase       ( self							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]				):
          if "\u4e00" <= char <= "\u9fff":
                return True
          return False
    def      _UpperCamelCase       ( self							:					int							,					SCREAMING_SNAKE_CASE__							:					Any				):
          if ("a" <= char <= "z") or ("A" <= char <= "Z"):
                return True
          return False
    def      _UpperCamelCase       ( self							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					Tuple				):
          if char in ",;:.?!~，；：。？！《》【】":
                return True
          return False
    def      _UpperCamelCase       ( self							:					List[Any]							,					SCREAMING_SNAKE_CASE__							:					int				):
          if char == " " or char == "\t" or char == "\n" or char == "\r":
                return True
          if len(SCREAMING_SNAKE_CASE__				) == 1:
                lowerCamelCase__           =		unicodedata.category(SCREAMING_SNAKE_CASE__				)
                if cat == "Zs":
                      return True
          return False
    def      _UpperCamelCase       ( self							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					int				):
          lowerCamelCase__           =		{}
          with io.open(SCREAMING_SNAKE_CASE__							,					'r'							,					encoding='utf-8'				) as f:
                for index, line in enumerate(SCREAMING_SNAKE_CASE__				):
                      lowerCamelCase__           =		line.rstrip('\n'				)
                      lowerCamelCase__           =		int(SCREAMING_SNAKE_CASE__				)

          return token_to_idx


    def      _UpperCamelCase       ( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					Optional[str] = None				):
          lowerCamelCase__           =		0
          if os.path.isdir(SCREAMING_SNAKE_CASE__				):
                lowerCamelCase__           =		os.path.join(
                    SCREAMING_SNAKE_CASE__							,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)
          else:
                lowerCamelCase__           =		(filename_prefix + '-' if filename_prefix else '') + save_directory
          with open(SCREAMING_SNAKE_CASE__							,					'w'							,					encoding='utf-8'				) as writer:
                for token, token_index in sorted(self.vocab.items()							,					key=lambda SCREAMING_SNAKE_CASE__				: kv[1]				):
                      if index != token_index:
                            logger.warning(
                                F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                                ' Please check that the vocabulary is not corrupted!'				)
                            lowerCamelCase__           =		token_index
                      writer.write(token + '\n'				)
                      index += 1

          lowerCamelCase__           =		os.path.join(SCREAMING_SNAKE_CASE__							,					'sentencepiece.bpe.model'				)
          with open(SCREAMING_SNAKE_CASE__							,					'wb'				) as fi:
                lowerCamelCase__           =		self.sp_model.serialized_model_proto()
                fi.write(SCREAMING_SNAKE_CASE__				)

          return (vocab_file,)







"""simple docstring"""


def   snake_case						(				_a:     list[list[float]]							)->		list[list[float]]:
      '''simple docstring'''


      lowerCamelCase__           =		[]
      for data in source_data:
            for i, el in enumerate(_a							):
                  if len(_a							) < i + 1:
                        data_lists.append([]							)
                  data_lists[i].append(float(_a							)							)
      return data_lists



def   snake_case						(				_a:     list[list[float]]		,   _a:     list[int]							)->		list[list[float]]:
      '''simple docstring'''


      lowerCamelCase__           =		[]
      for dlist, weight in zip(_a		,   _a							):
            lowerCamelCase__           =		min(_a							)
            lowerCamelCase__           =		max(_a							)

            lowerCamelCase__           =		[]
            # for weight 0 score is 1 - actual score
            if weight == 0:
                  for item in dlist:
                        try:
                              score.append(1 - ((item - mind) / (maxd - mind))							)
                        except ZeroDivisionError:
                              score.append(1							)

            elif weight == 1:
                  for item in dlist:
                        try:
                              score.append((item - mind) / (maxd - mind)							)
                        except ZeroDivisionError:
                              score.append(0							)

        # weight not 0 or 1
            else:
                  lowerCamelCase__           =		F'Invalid weight of {weight:f} provided'
                  raise ValueError(_a							)

            score_lists.append(_a							)

      return score_lists



def   snake_case						(				_a:     list[list[float]]							)->		list[float]:
      '''simple docstring'''


      lowerCamelCase__           =		[0 for i in range(len(score_lists[0]							)							)]

      for slist in score_lists:
            for j, ele in enumerate(_a							):
                  lowerCamelCase__           =		final_scores[j] + ele

      return final_scores






def   snake_case						(				_a:     list[list[float]]		,   _a:     list[int]							)->		list[list[float]]:
      '''simple docstring'''


      lowerCamelCase__           =		get_data(_a							)
      lowerCamelCase__           =		calculate_each_score(_a		,   _a							)
      lowerCamelCase__           =		generate_final_scores(_a							)

      # append scores to source data
      for i, ele in enumerate(_a							):
            source_data[i].append(_a							)

      return source_data



