



'''simple docstring'''




import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a     :    Dict      						=					logging.get_logger(__name__)

a     :    Optional[int]      						=					"""▁"""

a     :    str      						=					{"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

a     :    int      						=					{
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

a     :    Tuple      						=					{
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

a     :    Optional[int]      						=					{
    """ernie-m-base""": 5_1_4,
    """ernie-m-large""": 5_1_4,
}

a     :    Union[str, Any]      						=					{
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}

class     UpperCamelCase_				(							__A    ):
	lowercase    =				["input_ids"]

	lowercase    =				VOCAB_FILES_NAMES
	lowercase    =				PRETRAINED_INIT_CONFIGURATION
	lowercase    =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase    =				PRETRAINED_VOCAB_FILES_MAP
	lowercase    =				RESOURCE_FILES_NAMES





	def __init__(						self	,	A	,	A=None	,	A=False	,	A="utf8"	,	A="[UNK]"	,	A="[SEP]"	,	A="[PAD]"	,	A="[CLS]"	,	A="[MASK]"	,	A = None	,	**A	,	)    ->    List[str]:
								# Mask token behave like a normal word, i.e. include the space before it and
								# is included in the raw text, there should be a match in a non-normalized sentence.

								UpperCAmelCase  :	Optional[int]							  =	{} if sp_model_kwargs is None else sp_model_kwargs
								super().__init__(
								    do_lower_case=UpperCamelCase__	,	unk_token=UpperCamelCase__	,	sep_token=UpperCamelCase__	,	pad_token=UpperCamelCase__	,	cls_token=UpperCamelCase__	,	mask_token=UpperCamelCase__	,	vocab_file=UpperCamelCase__	,	encoding=UpperCamelCase__	,	sp_model_kwargs=self.sp_model_kwargs	,	**UpperCamelCase__	,	)
								UpperCAmelCase  :	str							  =	do_lower_case
								UpperCAmelCase  :	Optional[Any]							  =	sentencepiece_model_ckpt
								UpperCAmelCase  :	List[Any]							  =	spm.SentencePieceProcessor(**self.sp_model_kwargs					)
								self.sp_model.Load(UpperCamelCase__					)

								# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
								if vocab_file is not None:
															UpperCAmelCase  :	Optional[int]							  =	self.load_vocab(filepath=UpperCamelCase__					)
								else:
															UpperCAmelCase  :	Dict							  =	{self.sp_model.id_to_piece(UpperCamelCase__					): id for id in range(self.sp_model.get_piece_size()					)}
								UpperCAmelCase  :	Tuple							  =	{v: k for k, v in self.vocab.items()}





	def 				_lowercase(						self	,	A					)    ->    Tuple:
								if text is None:
															return None

								UpperCAmelCase  :	Optional[Any]							  =	self.tokenize(UpperCamelCase__					)
								UpperCAmelCase  :	Dict							  =	'', []

								for i, ch in enumerate(UpperCamelCase__					):
															if ch in self.SP_CHAR_MAPPING:
																						UpperCAmelCase  :	Any							  =	self.SP_CHAR_MAPPING.get(UpperCamelCase__					)
															else:
																						UpperCAmelCase  :	Tuple							  =	unicodedata.normalize("""NFKC"""	,	UpperCamelCase__					)
															if self.is_whitespace(UpperCamelCase__					):
																						continue
															normalized_text += ch
															char_mapping.extend([i] * len(UpperCamelCase__					)					)

								UpperCAmelCase  :	Tuple							  =	normalized_text, [], 0

								if self.do_lower_case:
															UpperCAmelCase  :	int							  =	text.lower()

								for token in split_tokens:
															if token[:1] == "▁":
																						UpperCAmelCase  :	Dict							  =	token[1:]
															UpperCAmelCase  :	List[str]							  =	text[offset:].index(UpperCamelCase__					) + offset
															UpperCAmelCase  :	Tuple							  =	start + len(UpperCamelCase__					)

															token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)					)
															UpperCAmelCase  :	List[Any]							  =	end
								return token_mapping





	@property
	def 				_lowercase(						self					)    ->    List[str]:
								return len(self.vocab					)





	def 				_lowercase(						self					)    ->    int:
								return dict(self.vocab	,	**self.added_tokens_encoder					)





	def __getstate__(						self					)    ->    Optional[Any]:
								UpperCAmelCase  :	Optional[int]							  =	self.__dict__.copy()
								UpperCAmelCase  :	int							  =	None
								return state





	def __setstate__(						self	,	A					)    ->    List[str]:
								UpperCAmelCase  :	List[Any]							  =	d

								# for backward compatibility
								if not hasattr(self	,	"""sp_model_kwargs"""					):
															UpperCAmelCase  :	Tuple							  =	{}

								UpperCAmelCase  :	List[str]							  =	spm.SentencePieceProcessor(**self.sp_model_kwargs					)
								self.sp_model.Load(self.sentencepiece_model_ckpt					)





	def 				_lowercase(						self	,	A					)    ->    Optional[int]:
								return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase__	,	UpperCamelCase__					) for c in text)					)





	def 				_lowercase(						self	,	A	,	A=False	,	A=64	,	A=0.1					)    ->    Union[str, Any]:

								if self.sp_model_kwargs.get("""enable_sampling"""					) is True:
															UpperCAmelCase  :	int							  =	True
								if self.sp_model_kwargs.get("""alpha"""					) is not None:
															UpperCAmelCase  :	Tuple							  =	self.sp_model_kwargs.get("""alpha"""					)
								if self.sp_model_kwargs.get("""nbest_size"""					) is not None:
															UpperCAmelCase  :	Dict							  =	self.sp_model_kwargs.get("""nbest_size"""					)

								if not enable_sampling:
															UpperCAmelCase  :	Optional[int]							  =	self.sp_model.EncodeAsPieces(UpperCamelCase__					)
								else:
															UpperCAmelCase  :	Dict							  =	self.sp_model.SampleEncodeAsPieces(UpperCamelCase__	,	UpperCamelCase__	,	UpperCamelCase__					)
								UpperCAmelCase  :	Any							  =	[]
								for pi, piece in enumerate(UpperCamelCase__					):
															if piece == SPIECE_UNDERLINE:
																						if not pieces[pi + 1].startswith(UpperCamelCase__					) and pi != 0:
																													new_pieces.append(UpperCamelCase__					)
																													continue
																						else:
																													continue
															UpperCAmelCase  :	Optional[Any]							  =	0
															for i, chunk in enumerate(UpperCamelCase__					):
																						if chunk == SPIECE_UNDERLINE:
																													continue
																						if self.is_ch_char(UpperCamelCase__					) or self.is_punct(UpperCamelCase__					):
																													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																				new_pieces.append(piece[lst_i:i]					)
																													new_pieces.append(UpperCamelCase__					)
																													UpperCAmelCase  :	List[Any]							  =	i + 1
																						elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																				new_pieces.append(piece[lst_i:i]					)
																													UpperCAmelCase  :	str							  =	i
																						elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																				new_pieces.append(piece[lst_i:i]					)
																													UpperCAmelCase  :	List[Any]							  =	i
															if len(UpperCamelCase__					) > lst_i:
																						new_pieces.append(piece[lst_i:]					)
								return new_pieces





	def 				_lowercase(						self	,	A					)    ->    List[Any]:
								UpperCAmelCase  :	Optional[int]							  =	''.join(UpperCamelCase__					).replace(UpperCamelCase__	,	""" """					).strip()
								return out_string





	def 				_lowercase(						self	,	A					)    ->    Optional[int]:
								UpperCAmelCase  :	Dict							  =	self.convert_ids_to_tokens(UpperCamelCase__					)
								UpperCAmelCase  :	int							  =	''.join(UpperCamelCase__					).replace(UpperCamelCase__	,	""" """					).strip()
								return out_string





	def 				_lowercase(						self	,	A					)    ->    str:
								return self.vocab.get(UpperCamelCase__	,	self.vocab.get(self.unk_token					)					)





	def 				_lowercase(						self	,	A					)    ->    Union[str, Any]:
								return self.reverse_vocab.get(UpperCamelCase__	,	self.unk_token					)





	def 				_lowercase(						self	,	A	,	A=None					)    ->    Any:
								if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								UpperCAmelCase  :	Optional[int]							  =	[self.cls_token_id]
								UpperCAmelCase  :	Optional[Any]							  =	[self.sep_token_id]
								return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep





	def 				_lowercase(						self	,	A	,	A=None					)    ->    Optional[Any]:
								if offset_mapping_a is None:
															return [(0, 0)] + offset_mapping_a + [(0, 0)]

								return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]





	def 				_lowercase(						self	,	A	,	A=None	,	A=False					)    ->    Tuple:

								if already_has_special_tokens:
															if token_ids_a is not None:
																						raise ValueError(
																						    """You should not supply a second sequence if the provided sequence of """
																						    """ids is already formatted with special tokens for the model."""					)
															return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

								if token_ids_a is not None:
															return [1] + ([0] * len(UpperCamelCase__					)) + [1, 1] + ([0] * len(UpperCamelCase__					)) + [1]
								return [1] + ([0] * len(UpperCamelCase__					)) + [1]





	def 				_lowercase(						self	,	A	,	A = None					)    ->    int:
								# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
								if token_ids_a is None:
															# [CLS] X [SEP]
															return (len(UpperCamelCase__					) + 2) * [0]

								# [CLS] A [SEP] [SEP] B [SEP]
								return [0] * (len(UpperCamelCase__					) + 1) + [1] * (len(UpperCamelCase__					) + 3)





	def 				_lowercase(						self	,	A					)    ->    Optional[int]:
								if "\u4e00" <= char <= "\u9fff":
															return True
								return False





	def 				_lowercase(						self	,	A					)    ->    List[str]:
								if ("a" <= char <= "z") or ("A" <= char <= "Z"):
															return True
								return False





	def 				_lowercase(						self	,	A					)    ->    List[str]:
								if char in ",;:.?!~，；：。？！《》【】":
															return True
								return False





	def 				_lowercase(						self	,	A					)    ->    List[str]:
								if char == " " or char == "\t" or char == "\n" or char == "\r":
															return True
								if len(UpperCamelCase__					) == 1:
															UpperCAmelCase  :	int							  =	unicodedata.category(UpperCamelCase__					)
															if cat == "Zs":
																						return True
								return False





	def 				_lowercase(						self	,	A					)    ->    Dict:
								UpperCAmelCase  :	Optional[Any]							  =	{}
								with io.open(UpperCamelCase__	,	"""r"""	,	encoding="""utf-8"""					) as f:
															for index, line in enumerate(UpperCamelCase__					):
																						UpperCAmelCase  :	Union[str, Any]							  =	line.rstrip("""\n"""					)
																						UpperCAmelCase  :	Tuple							  =	int(UpperCamelCase__					)

								return token_to_idx






	def 				_lowercase(						self	,	A	,	A = None					)    ->    List[str]:
								UpperCAmelCase  :	Union[str, Any]							  =	0
								if os.path.isdir(UpperCamelCase__					):
															UpperCAmelCase  :	int							  =	os.path.join(
															    UpperCamelCase__	,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
								else:
															UpperCAmelCase  :	List[Any]							  =	(filename_prefix + '-' if filename_prefix else '') + save_directory
								with open(UpperCamelCase__	,	"""w"""	,	encoding="""utf-8"""					) as writer:
															for token, token_index in sorted(self.vocab.items()	,	key=lambda A					: kv[1]					):
																						if index != token_index:
																													logger.warning(
																													    f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																													    """ Please check that the vocabulary is not corrupted!"""					)
																													UpperCAmelCase  :	str							  =	token_index
																						writer.write(token + """\n"""					)
																						index += 1

								UpperCAmelCase  :	Optional[Any]							  =	os.path.join(UpperCamelCase__	,	"""sentencepiece.bpe.model"""					)
								with open(UpperCamelCase__	,	"""wb"""					) as fi:
															UpperCAmelCase  :	int							  =	self.sp_model.serialized_model_proto()
															fi.write(UpperCamelCase__					)

								return (vocab_file,)






'''simple docstring'''




from typing import List, Union

import numpy as np

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, logging
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, ChunkPipeline


a     :    List[str]      						=					logging.get_logger(__name__)

class     UpperCamelCase_				(							__magic_name__    ):





	def 				_lowercase(						self	,	A					)    ->    Optional[int]:
								if isinstance(A	,	A					):
															UpperCAmelCase  :	Union[str, Any]							  =	[label.strip() for label in labels.split(""","""					) if label.strip()]
								return labels






	def __call__(						self	,	A	,	A	,	A					)    ->    str:
								if len(A					) == 0 or len(A					) == 0:
															raise ValueError("""You must include at least one label and at least one sequence."""					)
								if hypothesis_template.format(labels[0]					) == hypothesis_template:
															raise ValueError(
															    (
															        """The provided hypothesis_template \"{}\" was not able to be formatted with the target labels. """
															        """Make sure the passed template includes formatting syntax such as {{}} where the label should go."""
															    ).format(A					)					)

								if isinstance(A	,	A					):
															UpperCAmelCase  :	Tuple							  =	[sequences]

								UpperCAmelCase  :	Optional[Any]							  =	[]
								for sequence in sequences:
															sequence_pairs.extend([[sequence, hypothesis_template.format(A					)] for label in labels]					)

								return sequence_pairs, sequences




@add_end_docstrings(__magic_name__    )
class     UpperCamelCase_				(							__magic_name__    ):





	def __init__(						self	,	A=ZeroShotClassificationArgumentHandler()	,	*A	,	**A					)    ->    Optional[int]:
								UpperCAmelCase  :	Tuple							  =	args_parser
								super().__init__(*A	,	**A					)
								if self.entailment_id == -1:
															logger.warning(
															    """Failed to determine 'entailment' label id from the label2id mapping in the model config. Setting to """
															    """-1. Define a descriptive label2id mapping in the model config to ensure correct outputs."""					)





	@property
	def 				_lowercase(						self					)    ->    List[Any]:
								for label, ind in self.model.config.labelaid.items():
															if label.lower().startswith("""entail"""					):
																						return ind
								return -1





	def 				_lowercase(						self	,	A	,	A=True	,	A=True	,	A=TruncationStrategy.ONLY_FIRST	,	**A					)    ->    str:
								UpperCAmelCase  :	Tuple							  =	self.framework
								if self.tokenizer.pad_token is None:
															# Override for tokenizers not supporting padding
															logger.error(
															    """Tokenizer was not supporting padding necessary for zero-shot, attempting to use """
															    """ `pad_token=eos_token`"""					)
															UpperCAmelCase  :	Any							  =	self.tokenizer.eos_token
								try:
															UpperCAmelCase  :	Tuple							  =	self.tokenizer(
															    A	,	add_special_tokens=A	,	return_tensors=A	,	padding=A	,	truncation=A	,	)
								except Exception as e:
															if "too short" in str(A					):
																						# tokenizers might yell that we want to truncate
																						# to a value that is not even reached by the input.
																						# In that case we don't want to truncate.
																						# It seems there's not a really better way to catch that
																						# exception.

																						UpperCAmelCase  :	List[str]							  =	self.tokenizer(
																						    A	,	add_special_tokens=A	,	return_tensors=A	,	padding=A	,	truncation=TruncationStrategy.DO_NOT_TRUNCATE	,	)
															else:
																						raise e

								return inputs





	def 				_lowercase(						self	,	**A					)    ->    Tuple:
								if kwargs.get("""multi_class"""	,	A					) is not None:
															UpperCAmelCase  :	Any							  =	kwargs["""multi_class"""]
															logger.warning(
															    """The `multi_class` argument has been deprecated and renamed to `multi_label`. """
															    """`multi_class` will be removed in a future version of Transformers."""					)
								UpperCAmelCase  :	int							  =	{}
								if "candidate_labels" in kwargs:
															UpperCAmelCase  :	Tuple							  =	self._args_parser._parse_labels(kwargs["""candidate_labels"""]					)
								if "hypothesis_template" in kwargs:
															UpperCAmelCase  :	List[Any]							  =	kwargs["""hypothesis_template"""]

								UpperCAmelCase  :	Dict							  =	{}
								if "multi_label" in kwargs:
															UpperCAmelCase  :	Union[str, Any]							  =	kwargs["""multi_label"""]
								return preprocess_params, {}, postprocess_params





	def __call__(						self	,	A	,	*A	,	**A	,	)    ->    Tuple:
								if len(A					) == 0:
															pass
								elif len(A					) == 1 and "candidate_labels" not in kwargs:
															UpperCAmelCase  :	Optional[Any]							  =	args[0]
								else:
															raise ValueError(f'''Unable to understand extra arguments {args}'''					)

								return super().__call__(A	,	**A					)





	def 				_lowercase(						self	,	A	,	A=None	,	A="This example is {}."					)    ->    List[Any]:
								UpperCAmelCase  ,  UpperCAmelCase  :	List[Any]							  =	self._args_parser(A	,	A	,	A					)

								for i, (candidate_label, sequence_pair) in enumerate(zip(A	,	A					)					):
															UpperCAmelCase  :	Any							  =	self._parse_and_tokenize([sequence_pair]					)

															yield {
															    "candidate_label": candidate_label,
															    "sequence": sequences[0],
															    "is_last": i == len(A					) - 1,
															    **model_input,
															}





	def 				_lowercase(						self	,	A					)    ->    Optional[int]:
								UpperCAmelCase  :	Optional[Any]							  =	inputs["""candidate_label"""]
								UpperCAmelCase  :	Tuple							  =	inputs["""sequence"""]
								UpperCAmelCase  :	List[Any]							  =	{k: inputs[k] for k in self.tokenizer.model_input_names}
								UpperCAmelCase  :	Tuple							  =	self.model(**A					)

								UpperCAmelCase  :	Optional[int]							  =	{
								    """candidate_label""": candidate_label,
								    """sequence""": sequence,
								    """is_last""": inputs["""is_last"""],
								    **outputs,
								}
								return model_outputs






	def 				_lowercase(						self	,	A	,	A=False					)    ->    List[str]:
								UpperCAmelCase  :	Dict							  =	[outputs["""candidate_label"""] for outputs in model_outputs]
								UpperCAmelCase  :	List[Any]							  =	[outputs["""sequence"""] for outputs in model_outputs]
								UpperCAmelCase  :	List[Any]							  =	np.concatenate([output["""logits"""].numpy() for output in model_outputs]					)
								UpperCAmelCase  :	Optional[Any]							  =	logits.shape[0]
								UpperCAmelCase  :	int							  =	len(A					)
								UpperCAmelCase  :	List[Any]							  =	N // n
								UpperCAmelCase  :	int							  =	logits.reshape((num_sequences, n, -1)					)

								if multi_label or len(A					) == 1:
															# softmax over the entailment vs. contradiction dim for each label independently
															UpperCAmelCase  :	str							  =	self.entailment_id
															UpperCAmelCase  :	str							  =	-1 if entailment_id == 0 else 0
															UpperCAmelCase  :	Optional[Any]							  =	reshaped_outputs[..., [contradiction_id, entailment_id]]
															UpperCAmelCase  :	int							  =	np.exp(A					) / np.exp(A					).sum(-1	,	keepdims=A					)
															UpperCAmelCase  :	int							  =	scores[..., 1]
								else:
															# softmax the "entailment" logits over all candidate labels
															UpperCAmelCase  :	Dict							  =	reshaped_outputs[..., self.entailment_id]
															UpperCAmelCase  :	Optional[int]							  =	np.exp(A					) / np.exp(A					).sum(-1	,	keepdims=A					)

								UpperCAmelCase  :	int							  =	list(reversed(scores[0].argsort()					)					)
								return {
								    "sequence": sequences[0],
								    "labels": [candidate_labels[i] for i in top_inds],
								    "scores": scores[0, top_inds].tolist(),
								}

