






'''simple docstring'''




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_rembert import RemBertTokenizer
else:
		__lowerCamelCase   :					Any   							=  None

__lowerCamelCase   :					List[Any]   							=  logging.get_logger(__name__)
__lowerCamelCase   :					int   							=  {'vocab_file': 'sentencepiece.model', 'tokenizer_file': 'tokenizer.json'}

__lowerCamelCase   :					int   							=  {
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
    'tokenizer_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/tokenizer.json',
    },
}

__lowerCamelCase   :					Optional[Any]   							=  {
    'google/rembert': 256,
}

__lowerCamelCase   :					Tuple   							=  '▁'







class    A_	(__lowercase							):
							"""simple docstring"""




							a__   		=	VOCAB_FILES_NAMES
							a__   		=	PRETRAINED_VOCAB_FILES_MAP
							a__   		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							a__   		=	RemBertTokenizer

							def __init__(		self			:Union[str, Any]       ,							lowerCAmelCase__			:Tuple=None       ,							lowerCAmelCase__			:Any=None       ,							lowerCAmelCase__			:str=True       ,							lowerCAmelCase__			:List[str]=True       ,							lowerCAmelCase__			:str=False       ,							lowerCAmelCase__			:Tuple="[CLS]"       ,							lowerCAmelCase__			:Optional[Any]="[SEP]"       ,							lowerCAmelCase__			:Union[str, Any]="<unk>"       ,							lowerCAmelCase__			:List[str]="[SEP]"       ,							lowerCAmelCase__			:str="<pad>"       ,							lowerCAmelCase__			:str="[CLS]"       ,							lowerCAmelCase__			:List[str]="[MASK]"       ,							**lowerCAmelCase__			:Dict       ,							)				->  Optional[int]:


										'''simple docstring'''




										snake_case_				:					Dict   =       AddedToken(__a       ,							lstrip=__a       ,							rstrip=__a				) if isinstance(__a       ,							__a				) else mask_token

										super().__init__(
										    __a       ,							tokenizer_file=__a       ,							do_lower_case=__a       ,							remove_space=__a       ,							keep_accents=__a       ,							bos_token=__a       ,							eos_token=__a       ,							unk_token=__a       ,							sep_token=__a       ,							pad_token=__a       ,							cls_token=__a       ,							mask_token=__a       ,							**__a       ,							)

										snake_case_				:					Optional[Any]   =       do_lower_case
										snake_case_				:					str   =       remove_space
										snake_case_				:					Any   =       keep_accents
										snake_case_				:					Optional[Any]   =       vocab_file
										snake_case_				:					Optional[int]   =       False if not self.vocab_file else True

							def 							_A	(		self			:str       ,							lowerCAmelCase__			:List[int]       ,							lowerCAmelCase__			:Optional[List[int]] = None				)				->  List[int]:


										'''simple docstring'''




										snake_case_				:					Optional[Any]   =       [self.sep_token_id]
										snake_case_				:					Dict   =       [self.cls_token_id]
										if token_ids_a is None:
													return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep

							def 							_A	(		self			:Optional[Any]       ,							lowerCAmelCase__			:List[int]       ,							lowerCAmelCase__			:Optional[List[int]] = None       ,							lowerCAmelCase__			:bool = False				)				->  List[int]:


										'''simple docstring'''




										if already_has_special_tokens:
													if token_ids_a is not None:
																raise ValueError(
																    "You should not supply a second sequence if the provided sequence of "
																    "ids is already formatted with special tokens for the model."				)
													return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

										if token_ids_a is not None:
													return [1] + ([0] * len(__a				)) + [1] + ([0] * len(__a				)) + [1]
										return [1] + ([0] * len(__a				)) + [1]

							def 							_A	(		self			:Dict       ,							lowerCAmelCase__			:List[int]       ,							lowerCAmelCase__			:Optional[List[int]] = None				)				->  List[int]:


										'''simple docstring'''




										snake_case_				:					Union[str, Any]   =       [self.sep_token_id]
										snake_case_				:					List[str]   =       [self.cls_token_id]

										if token_ids_a is None:
													return len(cls + token_ids_a + sep				) * [0]
										return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






							def 							_A	(		self			:List[str]       ,							lowerCAmelCase__			:str       ,							lowerCAmelCase__			:Optional[str] = None				)				->  Tuple[str]:


										'''simple docstring'''




										if not os.path.isdir(__a				):
													logger.error("Vocabulary path ({}) should be a directory".format(__a				)				)
													return
										snake_case_				:					Optional[Any]   =       os.path.join(
										    __a       ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)

										if os.path.abspath(self.vocab_file				) != os.path.abspath(__a				):
													copyfile(self.vocab_file       ,							__a				)

										return (out_vocab_file,)












'''simple docstring'''




import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
__lowerCamelCase   :					List[str]   							=					{'''LayoutLMv2Config''', '''LayoutLMv3Config'''}







@is_pipeline_test
class    A_	(unittest.TestCase							):
							"""simple docstring"""




							a__   		=	MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
							a__   		=	TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

							if model_mapping is not None:
														a__   		=	{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
							if tf_model_mapping is not None:
														a__   		=	{
														    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
														}

							def 							_A	(		self			:Tuple       ,							lowerCAmelCase__			:Tuple       ,							lowerCAmelCase__			:Union[str, Any]       ,							lowerCAmelCase__			:Dict				)				->  Any:


										'''simple docstring'''




										snake_case_				:					Optional[Any]   =       ZeroShotClassificationPipeline(
										    model=lowerCAmelCase__       ,							tokenizer=lowerCAmelCase__       ,							candidate_labels=["polics", "health"]				)
										return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]

							def 							_A	(		self			:List[str]       ,							lowerCAmelCase__			:Optional[int]       ,							lowerCAmelCase__			:Any				)				->  Optional[Any]:


										'''simple docstring'''




										snake_case_				:					Tuple   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels="politics"				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										# No kwarg
										snake_case_				:					List[Any]   =       classifier("Who are you voting for in 2020?"       ,							["politics"]				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										snake_case_				:					Dict   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels=["politics"]				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										snake_case_				:					int   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels="politics, public health"				)
										self.assertEqual(
										    lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}				)
										self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]				)				)       ,							1.0				)

										snake_case_				:					Optional[int]   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health"]				)
										self.assertEqual(
										    lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}				)
										self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]				)				)       ,							1.0				)

										snake_case_				:					str   =       classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template="This text is about {}"				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										# https://github.com/huggingface/transformers/issues/13846
										snake_case_				:					Dict   =       classifier(["I am happy"]       ,							["positive", "negative"]				)
										self.assertEqual(
										    lowerCAmelCase__       ,							[
										        {"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}
										        for i in range(1				)
										    ]       ,							)
										snake_case_				:					Tuple   =       classifier(["I am happy", "I am sad"]       ,							["positive", "negative"]				)
										self.assertEqual(
										    lowerCAmelCase__       ,							[
										        {"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}
										        for i in range(2				)
										    ]       ,							)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(""       ,							candidate_labels="politics"				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(lowerCAmelCase__       ,							candidate_labels="politics"				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier("Who are you voting for in 2020?"       ,							candidate_labels=""				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier("Who are you voting for in 2020?"       ,							candidate_labels=lowerCAmelCase__				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(
													    "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template="Not formatting template"       ,							)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(
													    "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template=lowerCAmelCase__       ,							)

										self.run_entailment_id(lowerCAmelCase__				)

							def 							_A	(		self			:List[Any]       ,							lowerCAmelCase__			:Pipeline				)				->  Union[str, Any]:


										'''simple docstring'''




										snake_case_				:					int   =       zero_shot_classifier.model.config
										snake_case_				:					Optional[int]   =       config.labelaid
										snake_case_				:					Tuple   =       zero_shot_classifier.entailment_id

										snake_case_				:					Optional[Any]   =       {"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							-1				)

										snake_case_				:					Tuple   =       {"entailment": 0, "neutral": 1, "contradiction": 2}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							0				)

										snake_case_				:					str   =       {"ENTAIL": 0, "NON-ENTAIL": 1}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							0				)

										snake_case_				:					str   =       {"ENTAIL": 2, "NEUTRAL": 1, "CONTR": 0}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							2				)

										snake_case_				:					List[str]   =       original_labelaid
										self.assertEqual(lowerCAmelCase__       ,							zero_shot_classifier.entailment_id				)

							@require_torch
							def 							_A	(		self			:Tuple				)				->  Any:


										'''simple docstring'''




										snake_case_				:					List[Any]   =       pipeline(
										    "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="pt"       ,							)
										# There was a regression in 4.10 for this
										# Adding a test so we don't make the mistake again.
										# https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
										zero_shot_classifier(
										    "Who are you voting for in 2020?" * 100       ,							candidate_labels=["politics", "public health", "science"]				)

							@require_torch
							def 							_A	(		self			:Optional[Any]				)				->  Tuple:


										'''simple docstring'''




										snake_case_				:					Union[str, Any]   =       pipeline(
										    "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="pt"       ,							)
										snake_case_				:					int   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["science", "public health", "politics"],
										        "scores": [0.3_3_3, 0.3_3_3, 0.3_3_3],
										    }       ,							)

							@require_tf
							def 							_A	(		self			:Union[str, Any]				)				->  Dict:


										'''simple docstring'''




										snake_case_				:					List[str]   =       pipeline(
										    "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="tf"       ,							)
										snake_case_				:					Optional[int]   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["science", "public health", "politics"],
										        "scores": [0.3_3_3, 0.3_3_3, 0.3_3_3],
										    }       ,							)

							@slow
							@require_torch
							def 							_A	(		self			:Union[str, Any]				)				->  int:


										'''simple docstring'''




										snake_case_				:					int   =       pipeline("zero-shot-classification"       ,							model="roberta-large-mnli"       ,							framework="pt"				)
										snake_case_				:					str   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["politics", "public health", "science"],
										        "scores": [0.9_7_6, 0.0_1_5, 0.0_0_9],
										    }       ,							)
										snake_case_				:					Optional[int]   =       zero_shot_classifier(
										    "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
										    " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
										    " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
										    " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
										    " machine translation tasks show these models to be superior in quality while being more parallelizable"
										    " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
										    " English-to-German translation task, improving over the existing best results, including ensembles by"
										    " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
										    " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
										    " fraction of the training costs of the best models from the literature. We show that the Transformer"
										    " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
										    " large and limited training data."       ,							candidate_labels=["machine learning", "statistics", "translation", "vision"]       ,							multi_label=lowerCAmelCase__       ,							)
										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": (
										            "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
										            " networks in an encoder-decoder configuration. The best performing models also connect the"
										            " encoder and decoder through an attention mechanism. We propose a new simple network"
										            " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
										            " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
										            " superior in quality while being more parallelizable and requiring significantly less time to"
										            " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
										            " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
										            " English-to-French translation task, our model establishes a new single-model state-of-the-art"
										            " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
										            " costs of the best models from the literature. We show that the Transformer generalizes well to"
										            " other tasks by applying it successfully to English constituency parsing both with large and"
										            " limited training data."
										        ),
										        "labels": ["translation", "machine learning", "vision", "statistics"],
										        "scores": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
										    }       ,							)






							@slow
							@require_tf
							def 							_A	(		self			:List[str]				)				->  str:


										'''simple docstring'''




										snake_case_				:					int   =       pipeline("zero-shot-classification"       ,							model="roberta-large-mnli"       ,							framework="tf"				)
										snake_case_				:					Optional[Any]   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["politics", "public health", "science"],
										        "scores": [0.9_7_6, 0.0_1_5, 0.0_0_9],
										    }       ,							)
										snake_case_				:					Tuple   =       zero_shot_classifier(
										    "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
										    " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
										    " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
										    " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
										    " machine translation tasks show these models to be superior in quality while being more parallelizable"
										    " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
										    " English-to-German translation task, improving over the existing best results, including ensembles by"
										    " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
										    " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
										    " fraction of the training costs of the best models from the literature. We show that the Transformer"
										    " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
										    " large and limited training data."       ,							candidate_labels=["machine learning", "statistics", "translation", "vision"]       ,							multi_label=lowerCAmelCase__       ,							)
										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": (
										            "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
										            " networks in an encoder-decoder configuration. The best performing models also connect the"
										            " encoder and decoder through an attention mechanism. We propose a new simple network"
										            " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
										            " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
										            " superior in quality while being more parallelizable and requiring significantly less time to"
										            " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
										            " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
										            " English-to-French translation task, our model establishes a new single-model state-of-the-art"
										            " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
										            " costs of the best models from the literature. We show that the Transformer generalizes well to"
										            " other tasks by applying it successfully to English constituency parsing both with large and"
										            " limited training data."
										        ),
										        "labels": ["translation", "machine learning", "vision", "statistics"],
										        "scores": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
										    }       ,							)




