






'''simple docstring'''




import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
__lowerCamelCase   :					List[str]   							=					{'''LayoutLMv2Config''', '''LayoutLMv3Config'''}







@is_pipeline_test
class    A_	(unittest.TestCase							):
							"""simple docstring"""




							a__   		=	MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
							a__   		=	TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

							if model_mapping is not None:
														a__   		=	{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
							if tf_model_mapping is not None:
														a__   		=	{
														    config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
														}

							def 							_A	(		self			:Tuple       ,							lowerCAmelCase__			:Tuple       ,							lowerCAmelCase__			:Union[str, Any]       ,							lowerCAmelCase__			:Dict				)				->  Any:


										'''simple docstring'''




										snake_case_				:					Optional[Any]   =       ZeroShotClassificationPipeline(
										    model=lowerCAmelCase__       ,							tokenizer=lowerCAmelCase__       ,							candidate_labels=["polics", "health"]				)
										return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]

							def 							_A	(		self			:List[str]       ,							lowerCAmelCase__			:Optional[int]       ,							lowerCAmelCase__			:Any				)				->  Optional[Any]:


										'''simple docstring'''




										snake_case_				:					Tuple   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels="politics"				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										# No kwarg
										snake_case_				:					List[Any]   =       classifier("Who are you voting for in 2020?"       ,							["politics"]				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										snake_case_				:					Dict   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels=["politics"]				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										snake_case_				:					int   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels="politics, public health"				)
										self.assertEqual(
										    lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}				)
										self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]				)				)       ,							1.0				)

										snake_case_				:					Optional[int]   =       classifier("Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health"]				)
										self.assertEqual(
										    lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}				)
										self.assertAlmostEqual(sum(nested_simplify(outputs["scores"]				)				)       ,							1.0				)

										snake_case_				:					str   =       classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template="This text is about {}"				)
										self.assertEqual(lowerCAmelCase__       ,							{"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				)]}				)

										# https://github.com/huggingface/transformers/issues/13846
										snake_case_				:					Dict   =       classifier(["I am happy"]       ,							["positive", "negative"]				)
										self.assertEqual(
										    lowerCAmelCase__       ,							[
										        {"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}
										        for i in range(1				)
										    ]       ,							)
										snake_case_				:					Tuple   =       classifier(["I am happy", "I am sad"]       ,							["positive", "negative"]				)
										self.assertEqual(
										    lowerCAmelCase__       ,							[
										        {"sequence": ANY(lowerCAmelCase__				), "labels": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)], "scores": [ANY(lowerCAmelCase__				), ANY(lowerCAmelCase__				)]}
										        for i in range(2				)
										    ]       ,							)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(""       ,							candidate_labels="politics"				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(lowerCAmelCase__       ,							candidate_labels="politics"				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier("Who are you voting for in 2020?"       ,							candidate_labels=""				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier("Who are you voting for in 2020?"       ,							candidate_labels=lowerCAmelCase__				)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(
													    "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template="Not formatting template"       ,							)

										with self.assertRaises(lowerCAmelCase__				):
													classifier(
													    "Who are you voting for in 2020?"       ,							candidate_labels="politics"       ,							hypothesis_template=lowerCAmelCase__       ,							)

										self.run_entailment_id(lowerCAmelCase__				)

							def 							_A	(		self			:List[Any]       ,							lowerCAmelCase__			:Pipeline				)				->  Union[str, Any]:


										'''simple docstring'''




										snake_case_				:					int   =       zero_shot_classifier.model.config
										snake_case_				:					Optional[int]   =       config.labelaid
										snake_case_				:					Tuple   =       zero_shot_classifier.entailment_id

										snake_case_				:					Optional[Any]   =       {"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							-1				)

										snake_case_				:					Tuple   =       {"entailment": 0, "neutral": 1, "contradiction": 2}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							0				)

										snake_case_				:					str   =       {"ENTAIL": 0, "NON-ENTAIL": 1}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							0				)

										snake_case_				:					str   =       {"ENTAIL": 2, "NEUTRAL": 1, "CONTR": 0}
										self.assertEqual(zero_shot_classifier.entailment_id       ,							2				)

										snake_case_				:					List[str]   =       original_labelaid
										self.assertEqual(lowerCAmelCase__       ,							zero_shot_classifier.entailment_id				)

							@require_torch
							def 							_A	(		self			:Tuple				)				->  Any:


										'''simple docstring'''




										snake_case_				:					List[Any]   =       pipeline(
										    "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="pt"       ,							)
										# There was a regression in 4.10 for this
										# Adding a test so we don't make the mistake again.
										# https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
										zero_shot_classifier(
										    "Who are you voting for in 2020?" * 100       ,							candidate_labels=["politics", "public health", "science"]				)

							@require_torch
							def 							_A	(		self			:Optional[Any]				)				->  Tuple:


										'''simple docstring'''




										snake_case_				:					Union[str, Any]   =       pipeline(
										    "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="pt"       ,							)
										snake_case_				:					int   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["science", "public health", "politics"],
										        "scores": [0.3_3_3, 0.3_3_3, 0.3_3_3],
										    }       ,							)

							@require_tf
							def 							_A	(		self			:Union[str, Any]				)				->  Dict:


										'''simple docstring'''




										snake_case_				:					List[str]   =       pipeline(
										    "zero-shot-classification"       ,							model="sshleifer/tiny-distilbert-base-cased-distilled-squad"       ,							framework="tf"       ,							)
										snake_case_				:					Optional[int]   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["science", "public health", "politics"],
										        "scores": [0.3_3_3, 0.3_3_3, 0.3_3_3],
										    }       ,							)

							@slow
							@require_torch
							def 							_A	(		self			:Union[str, Any]				)				->  int:


										'''simple docstring'''




										snake_case_				:					int   =       pipeline("zero-shot-classification"       ,							model="roberta-large-mnli"       ,							framework="pt"				)
										snake_case_				:					str   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["politics", "public health", "science"],
										        "scores": [0.9_7_6, 0.0_1_5, 0.0_0_9],
										    }       ,							)
										snake_case_				:					Optional[int]   =       zero_shot_classifier(
										    "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
										    " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
										    " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
										    " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
										    " machine translation tasks show these models to be superior in quality while being more parallelizable"
										    " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
										    " English-to-German translation task, improving over the existing best results, including ensembles by"
										    " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
										    " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
										    " fraction of the training costs of the best models from the literature. We show that the Transformer"
										    " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
										    " large and limited training data."       ,							candidate_labels=["machine learning", "statistics", "translation", "vision"]       ,							multi_label=lowerCAmelCase__       ,							)
										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": (
										            "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
										            " networks in an encoder-decoder configuration. The best performing models also connect the"
										            " encoder and decoder through an attention mechanism. We propose a new simple network"
										            " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
										            " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
										            " superior in quality while being more parallelizable and requiring significantly less time to"
										            " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
										            " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
										            " English-to-French translation task, our model establishes a new single-model state-of-the-art"
										            " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
										            " costs of the best models from the literature. We show that the Transformer generalizes well to"
										            " other tasks by applying it successfully to English constituency parsing both with large and"
										            " limited training data."
										        ),
										        "labels": ["translation", "machine learning", "vision", "statistics"],
										        "scores": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
										    }       ,							)






							@slow
							@require_tf
							def 							_A	(		self			:List[str]				)				->  str:


										'''simple docstring'''




										snake_case_				:					int   =       pipeline("zero-shot-classification"       ,							model="roberta-large-mnli"       ,							framework="tf"				)
										snake_case_				:					Optional[Any]   =       zero_shot_classifier(
										    "Who are you voting for in 2020?"       ,							candidate_labels=["politics", "public health", "science"]				)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": "Who are you voting for in 2020?",
										        "labels": ["politics", "public health", "science"],
										        "scores": [0.9_7_6, 0.0_1_5, 0.0_0_9],
										    }       ,							)
										snake_case_				:					Tuple   =       zero_shot_classifier(
										    "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks"
										    " in an encoder-decoder configuration. The best performing models also connect the encoder and decoder"
										    " through an attention mechanism. We propose a new simple network architecture, the Transformer, based"
										    " solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two"
										    " machine translation tasks show these models to be superior in quality while being more parallelizable"
										    " and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014"
										    " English-to-German translation task, improving over the existing best results, including ensembles by"
										    " over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new"
										    " single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small"
										    " fraction of the training costs of the best models from the literature. We show that the Transformer"
										    " generalizes well to other tasks by applying it successfully to English constituency parsing both with"
										    " large and limited training data."       ,							candidate_labels=["machine learning", "statistics", "translation", "vision"]       ,							multi_label=lowerCAmelCase__       ,							)
										self.assertEqual(
										    nested_simplify(lowerCAmelCase__				)       ,							{
										        "sequence": (
										            "The dominant sequence transduction models are based on complex recurrent or convolutional neural"
										            " networks in an encoder-decoder configuration. The best performing models also connect the"
										            " encoder and decoder through an attention mechanism. We propose a new simple network"
										            " architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence"
										            " and convolutions entirely. Experiments on two machine translation tasks show these models to be"
										            " superior in quality while being more parallelizable and requiring significantly less time to"
										            " train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,"
										            " improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014"
										            " English-to-French translation task, our model establishes a new single-model state-of-the-art"
										            " BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training"
										            " costs of the best models from the literature. We show that the Transformer generalizes well to"
										            " other tasks by applying it successfully to English constituency parsing both with large and"
										            " limited training data."
										        ),
										        "labels": ["translation", "machine learning", "vision", "statistics"],
										        "scores": [0.8_1_7, 0.7_1_3, 0.0_1_8, 0.0_1_8],
										    }       ,							)












'''simple docstring'''




import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse('''1.0.0a'''):
		raise Exception('''requires fairseq >= 1.0.0a''')

logging.set_verbosity_info()
__lowerCamelCase   :					Union[str, Any]   							=					logging.get_logger(__name__)

__lowerCamelCase   :					Union[str, Any]   							=					'''Hello world! cécé herlolip'''

def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__     ,__magic_name__						)->    Optional[Any]:



			"""simple docstring"""



			snake_case_				:					str   =       FairseqRobertaModel.from_pretrained(__magic_name__						)
			roberta.eval()  # disable dropout
			snake_case_				:					Dict   =       roberta.model.encoder.sentence_encoder
			snake_case_				:					List[str]   =       XLMRobertaConfig(
			    vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings     ,hidden_size=roberta.cfg.model.encoder_embed_dim     ,num_hidden_layers=roberta.cfg.model.encoder_layers     ,num_attention_heads=roberta.cfg.model.encoder_attention_heads     ,intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim     ,max_position_embeddings=514     ,type_vocab_size=1     ,layer_norm_eps=1E-5     ,)
			if classification_head:
						snake_case_				:					List[str]   =       roberta.model.classification_heads["mnli"].out_proj.weight.shape[0]

			print("Our RoBERTa config:"     ,__magic_name__						)

			snake_case_				:					List[str]   =       XLMRobertaXLForSequenceClassification(__magic_name__						) if classification_head else XLMRobertaXLForMaskedLM(__magic_name__						)
			model.eval()

			# Now let's copy all the weights.
			# Embeddings
			snake_case_				:					List[Any]   =       roberta_sent_encoder.embed_tokens.weight
			snake_case_				:					int   =       roberta_sent_encoder.embed_positions.weight
			snake_case_				:					Union[str, Any]   =       torch.zeros_like(
			    model.roberta.embeddings.token_type_embeddings.weight						)  # just zero them out b/c RoBERTa doesn't use them.

			snake_case_				:					Union[str, Any]   =       roberta_sent_encoder.layer_norm.weight
			snake_case_				:					str   =       roberta_sent_encoder.layer_norm.bias

			for i in range(config.num_hidden_layers						):
						# Encoder: start of layer
						snake_case_				:					BertLayer   =       model.roberta.encoder.layer[i]
						snake_case_				:					TransformerSentenceEncoderLayer   =       roberta_sent_encoder.layers[i]

						snake_case_				:					RobertaAttention   =       layer.attention
						snake_case_				:					Dict   =       roberta_layer.self_attn_layer_norm.weight
						snake_case_				:					Dict   =       roberta_layer.self_attn_layer_norm.bias

						# self attention
						snake_case_				:					BertSelfAttention   =       layer.attention.self
						assert (
						    roberta_layer.self_attn.k_proj.weight.data.shape
						    == roberta_layer.self_attn.q_proj.weight.data.shape
						    == roberta_layer.self_attn.v_proj.weight.data.shape
						    == torch.Size((config.hidden_size, config.hidden_size)						)
						)

						snake_case_				:					Dict   =       roberta_layer.self_attn.q_proj.weight
						snake_case_				:					Any   =       roberta_layer.self_attn.q_proj.bias
						snake_case_				:					Optional[Any]   =       roberta_layer.self_attn.k_proj.weight
						snake_case_				:					Optional[Any]   =       roberta_layer.self_attn.k_proj.bias
						snake_case_				:					Optional[int]   =       roberta_layer.self_attn.v_proj.weight
						snake_case_				:					Any   =       roberta_layer.self_attn.v_proj.bias

						# self-attention output
						snake_case_				:					BertSelfOutput   =       layer.attention.output
						assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
						snake_case_				:					List[str]   =       roberta_layer.self_attn.out_proj.weight
						snake_case_				:					Optional[int]   =       roberta_layer.self_attn.out_proj.bias

						# this one is final layer norm
						snake_case_				:					int   =       roberta_layer.final_layer_norm.weight
						snake_case_				:					Union[str, Any]   =       roberta_layer.final_layer_norm.bias

						# intermediate
						snake_case_				:					BertIntermediate   =       layer.intermediate
						assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
						snake_case_				:					List[str]   =       roberta_layer.fca.weight
						snake_case_				:					List[Any]   =       roberta_layer.fca.bias

						# output
						snake_case_				:					BertOutput   =       layer.output
						assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
						snake_case_				:					Any   =       roberta_layer.fca.weight
						snake_case_				:					Any   =       roberta_layer.fca.bias
						# end of layer

			if classification_head:
						snake_case_				:					int   =       roberta.model.classification_heads["mnli"].dense.weight
						snake_case_				:					Union[str, Any]   =       roberta.model.classification_heads["mnli"].dense.bias
						snake_case_				:					Tuple   =       roberta.model.classification_heads["mnli"].out_proj.weight
						snake_case_				:					str   =       roberta.model.classification_heads["mnli"].out_proj.bias
			else:
						# LM Head
						snake_case_				:					Optional[Any]   =       roberta.model.encoder.lm_head.dense.weight
						snake_case_				:					int   =       roberta.model.encoder.lm_head.dense.bias
						snake_case_				:					Optional[Any]   =       roberta.model.encoder.lm_head.layer_norm.weight
						snake_case_				:					Optional[int]   =       roberta.model.encoder.lm_head.layer_norm.bias
						snake_case_				:					int   =       roberta.model.encoder.lm_head.weight
						snake_case_				:					List[str]   =       roberta.model.encoder.lm_head.bias

			# Let's check that we get the same results.
			snake_case_				:					torch.Tensor   =       roberta.encode(__magic_name__						).unsqueeze(0						)  # batch of size 1

			snake_case_				:					Union[str, Any]   =       model(__magic_name__						)[0]
			if classification_head:
						snake_case_				:					Optional[Any]   =       roberta.model.classification_heads["mnli"](roberta.extract_features(__magic_name__						)						)
			else:
						snake_case_				:					List[str]   =       roberta.model(__magic_name__						)[0]
			print(our_output.shape     ,their_output.shape						)
			snake_case_				:					str   =       torch.max(torch.abs(our_output - their_output						)						).item()
			print(F'''max_absolute_diff = {max_absolute_diff}'''						)  # ~ 1e-7
			snake_case_				:					Any   =       torch.allclose(__magic_name__     ,__magic_name__     ,atol=1E-3						)
			print("Do both models output the same tensors?"     ,"🔥" if success else "💩"						)
			if not success:
						raise Exception("Something went wRoNg"						)

			pathlib.Path(__magic_name__						).mkdir(parents=__magic_name__     ,exist_ok=__magic_name__						)
			print(F'''Saving model to {pytorch_dump_folder_path}'''						)
			model.save_pretrained(__magic_name__						)


if __name__ == "__main__":
		__lowerCamelCase   :					Any   							=					argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '''--roberta_checkpoint_path''', default=None, type=str, required=True, help='''Path the official PyTorch dump.'''
		)
		parser.add_argument(
		    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
		)
		parser.add_argument(
		    '''--classification_head''', action='''store_true''', help='''Whether to convert a final classification head.'''
		)
		__lowerCamelCase   :					Tuple   							=					parser.parse_args()
		convert_xlm_roberta_xl_checkpoint_to_pytorch(
		    args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
		)




