




import argparse
from pathlib import Path

import fairseq
import torch
from fairseq.models.xmod import XMODModel as FairseqXmodModel
from packaging import version

from transformers import XmodConfig, XmodForMaskedLM, XmodForSequenceClassification
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("0.12.2"):
 raise Exception("requires fairseq >= 0.12.2")
if version.parse(fairseq.__version__) > version.parse("2"):
 raise Exception("requires fairseq < v2")

logging.set_verbosity_info()
lowerCamelCase       :	Any									=     logging.get_logger(__name__)

lowerCamelCase       :	Tuple									=     "Hello, World!"
lowerCamelCase       :	Union[str, Any]									=     "en_XX"

def     _SCREAMING_SNAKE_CASE    (							lowercase			: str			,							lowercase			: str			,							lowercase			: bool  ):


      '''simple docstring'''



      lowerCamelCase_	  =					Path('data_bin'  )
      lowerCamelCase_	  =					FairseqXmodModel.from_pretrained(
          model_name_or_path=str(Path(lowercase  ).parent  )			,							checkpoint_file=Path(lowercase  ).name			,							_name='xmod_base'			,							arch='xmod_base'			,							task='multilingual_masked_lm'			,							data_name_or_path=str(lowercase  )			,							bpe='sentencepiece'			,							sentencepiece_model=str(Path(lowercase  ).parent / 'sentencepiece.bpe.model'  )			,							src_dict=str(data_dir / 'dict.txt'  )			,							)
      xmod.eval()  # disable dropout
      print(lowercase  )

      lowerCamelCase_	  =					xmod.model.encoder.sentence_encoder
      lowerCamelCase_	  =					XmodConfig(
          vocab_size=xmod_sent_encoder.embed_tokens.num_embeddings			,							hidden_size=xmod.cfg.model.encoder_embed_dim			,							num_hidden_layers=xmod.cfg.model.encoder_layers			,							num_attention_heads=xmod.cfg.model.encoder_attention_heads			,							intermediate_size=xmod.cfg.model.encoder_ffn_embed_dim			,							max_position_embeddings=5_14			,							type_vocab_size=1			,							layer_norm_eps=1e-5			,							pre_norm=xmod.cfg.model.encoder_normalize_before			,							adapter_reduction_factor=getattr(xmod.cfg.model			,							'bottleneck'			,							2  )			,							adapter_layer_norm=xmod.cfg.model.adapter_layer_norm			,							adapter_reuse_layer_norm=xmod.cfg.model.adapter_reuse_layer_norm			,							ln_before_adapter=xmod.cfg.model.ln_before_adapter			,							languages=xmod.cfg.model.languages			,							)
      if classification_head:
            lowerCamelCase_	  =					xmod.model.classification_heads['mnli'].out_proj.weight.shape[0]

      print('Our X-MOD config:'			,							lowercase  )

      lowerCamelCase_	  =					XmodForSequenceClassification(lowercase  ) if classification_head else XmodForMaskedLM(lowercase  )
      model.eval()

      # Now let's copy all the weights.
      # Embeddings
      lowerCamelCase_	  =					xmod_sent_encoder.embed_tokens.weight
      lowerCamelCase_	  =					xmod_sent_encoder.embed_positions.weight
      lowerCamelCase_	  =					torch.zeros_like(
          model.roberta.embeddings.token_type_embeddings.weight  )  # just zero them out b/c xmod doesn't use them.

      lowerCamelCase_	  =					xmod_sent_encoder.layernorm_embedding.weight
      lowerCamelCase_	  =					xmod_sent_encoder.layernorm_embedding.bias

      for i in range(config.num_hidden_layers  ):
            # Encoder: start of layer
            lowerCamelCase_	  =					model.roberta.encoder.layer[i]
            lowerCamelCase_	  =					xmod_sent_encoder.layers[i]

            # self attention
            lowerCamelCase_	  =					layer.attention.self
            if not (
                xmod_layer.self_attn.k_proj.weight.data.shape
                == xmod_layer.self_attn.q_proj.weight.data.shape
                == xmod_layer.self_attn.v_proj.weight.data.shape
                == torch.Size((config.hidden_size, config.hidden_size)  )
            ):
                  raise AssertionError('Dimensions of self-attention weights do not match.'  )

            lowerCamelCase_	  =					xmod_layer.self_attn.q_proj.weight
            lowerCamelCase_	  =					xmod_layer.self_attn.q_proj.bias
            lowerCamelCase_	  =					xmod_layer.self_attn.k_proj.weight
            lowerCamelCase_	  =					xmod_layer.self_attn.k_proj.bias
            lowerCamelCase_	  =					xmod_layer.self_attn.v_proj.weight
            lowerCamelCase_	  =					xmod_layer.self_attn.v_proj.bias

            # self-attention output
            lowerCamelCase_	  =					layer.attention.output
            if self_output.dense.weight.shape != xmod_layer.self_attn.out_proj.weight.shape:
                  raise AssertionError('Dimensions of self-attention output weights do not match.'  )
            lowerCamelCase_	  =					xmod_layer.self_attn.out_proj.weight
            lowerCamelCase_	  =					xmod_layer.self_attn.out_proj.bias
            lowerCamelCase_	  =					xmod_layer.self_attn_layer_norm.weight
            lowerCamelCase_	  =					xmod_layer.self_attn_layer_norm.bias

            # intermediate
            lowerCamelCase_	  =					layer.intermediate
            if intermediate.dense.weight.shape != xmod_layer.fca.weight.shape:
                  raise AssertionError('Dimensions of intermediate weights do not match.'  )
            lowerCamelCase_	  =					xmod_layer.fca.weight
            lowerCamelCase_	  =					xmod_layer.fca.bias

            # output
            lowerCamelCase_	  =					layer.output
            if bert_output.dense.weight.shape != xmod_layer.fca.weight.shape:
                  raise AssertionError('Dimensions of feed-forward weights do not match.'  )
            lowerCamelCase_	  =					xmod_layer.fca.weight
            lowerCamelCase_	  =					xmod_layer.fca.bias
            lowerCamelCase_	  =					xmod_layer.final_layer_norm.weight
            lowerCamelCase_	  =					xmod_layer.final_layer_norm.bias
            if bert_output.adapter_layer_norm is not None:
                  lowerCamelCase_	  =					xmod_layer.adapter_layer_norm.weight
                  lowerCamelCase_	  =					xmod_layer.adapter_layer_norm.bias

            if sorted(bert_output.adapter_modules.keys()  ) != sorted(xmod_layer.adapter_modules.keys()  ):
                  raise AssertionError('Lists of language adapters do not match.'  )
            for lang_code, adapter in xmod_layer.adapter_modules.items():
                  lowerCamelCase_	  =					bert_output.adapter_modules[lang_code]
                  lowerCamelCase_	  =					xmod_layer.adapter_modules[lang_code]
                  lowerCamelCase_	  =					from_adapter.fca.weight
                  lowerCamelCase_	  =					from_adapter.fca.bias
                  lowerCamelCase_	  =					from_adapter.fca.weight
                  lowerCamelCase_	  =					from_adapter.fca.bias

            # end of layer

      if xmod_sent_encoder.layer_norm is not None:
            lowerCamelCase_	  =					xmod_sent_encoder.layer_norm.weight
            lowerCamelCase_	  =					xmod_sent_encoder.layer_norm.bias

      if classification_head:
            lowerCamelCase_	  =					xmod.model.classification_heads['mnli'].dense.weight
            lowerCamelCase_	  =					xmod.model.classification_heads['mnli'].dense.bias
            lowerCamelCase_	  =					xmod.model.classification_heads['mnli'].out_proj.weight
            lowerCamelCase_	  =					xmod.model.classification_heads['mnli'].out_proj.bias
      else:
            # LM Head
            lowerCamelCase_	  =					xmod.model.encoder.lm_head.dense.weight
            lowerCamelCase_	  =					xmod.model.encoder.lm_head.dense.bias
            lowerCamelCase_	  =					xmod.model.encoder.lm_head.layer_norm.weight
            lowerCamelCase_	  =					xmod.model.encoder.lm_head.layer_norm.bias
            lowerCamelCase_	  =					xmod.model.encoder.lm_head.weight
            lowerCamelCase_	  =					xmod.model.encoder.lm_head.bias

      # Let's check that we get the same results.
      lowerCamelCase_	  =					xmod.encode(lowercase  ).unsqueeze(0  )  # batch of size 1
      model.roberta.set_default_language(lowercase  )

      lowerCamelCase_	  =					model(lowercase  )[0]
      if classification_head:
            lowerCamelCase_	  =					xmod.model.classification_heads['mnli'](xmod.extract_features(lowercase  )  )
      else:
            lowerCamelCase_	  =					xmod.model(lowercase			,							lang_id=[SAMPLE_LANGUAGE]  )[0]
      print(our_output.shape			,							their_output.shape  )
      lowerCamelCase_	  =					torch.max(torch.abs(our_output - their_output  )  ).item()
      print(f"""max_absolute_diff = {max_absolute_diff}"""  )  # ~ 1e-7
      lowerCamelCase_	  =					torch.allclose(lowercase			,							lowercase			,							atol=1e-3  )
      print('Do both models output the same tensors?'			,							'🔥' if success else '💩'  )
      if not success:
            raise Exception('Something went wRoNg'  )

      Path(lowercase  ).mkdir(parents=lowercase			,							exist_ok=lowercase  )
      print(f"""Saving model to {pytorch_dump_folder_path}"""  )
      model.save_pretrained(lowercase  )


if __name__ == "__main__":
 lowerCamelCase       :	Dict									=     argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     "--xmod_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
 )
 parser.add_argument(
     "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
 )
 parser.add_argument(
     "--classification_head", action="store_true", help="Whether to convert a final classification head."
 )
 lowerCamelCase       :	List[Any]									=     parser.parse_args()
 convert_xmod_checkpoint_to_pytorch(
     args.xmod_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
 )









from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
 from transformers import (
     TFAutoModel,
     TFAutoModelForCausalLM,
     TFAutoModelForMaskedLM,
     TFAutoModelForPreTraining,
     TFAutoModelForQuestionAnswering,
     TFAutoModelForSeqaSeqLM,
     TFAutoModelForSequenceClassification,
     TFAutoModelForTableQuestionAnswering,
     TFAutoModelForTokenClassification,
     TFAutoModelWithLMHead,
     TFBertForMaskedLM,
     TFBertForPreTraining,
     TFBertForQuestionAnswering,
     TFBertForSequenceClassification,
     TFBertModel,
     TFFunnelBaseModel,
     TFFunnelModel,
     TFGPTaLMHeadModel,
     TFRobertaForMaskedLM,
     TFTaForConditionalGeneration,
     TFTapasForQuestionAnswering,
 )
 from transformers.models.auto.modeling_tf_auto import (
     TF_MODEL_FOR_CAUSAL_LM_MAPPING,
     TF_MODEL_FOR_MASKED_LM_MAPPING,
     TF_MODEL_FOR_PRETRAINING_MAPPING,
     TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
     TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
     TF_MODEL_MAPPING,
 )
 from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 UpperCamelCase		     =		'''new-model'''


if is_tf_available():







 class 						A(		UpperCamelCase     ):



  '''simple docstring'''


  UpperCamelCase		     =		NewModelConfig







@require_tf
class 						A(		unittest.TestCase     ):



 '''simple docstring'''


 @slow
 def        a__				(       self      :						str						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					'bert-base-cased'
       lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

       lowerCamelCase_	  =					TFAutoModel.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						List[Any]						)     ->			Dict:

       """simple docstring"""






       lowerCamelCase_	  =					'bert-base-cased'
       lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

       lowerCamelCase_	  =					TFAutoModelForPreTraining.from_pretrained(A_						)
       self.assertIsNotNone(A_						)
       self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						Union[str, Any]						)     ->			str:

       """simple docstring"""






       for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForCausalLM.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForCausalLM.from_pretrained(A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						List[Any]						)     ->			Dict:

       """simple docstring"""






       for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelWithLMHead.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						int						)     ->			str:

       """simple docstring"""






       for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForMaskedLM.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForMaskedLM.from_pretrained(A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						Any						)     ->			List[Any]:

       """simple docstring"""






       for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForSeqaSeqLM.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForSeqaSeqLM.from_pretrained(A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						Tuple						)     ->			str:

       """simple docstring"""






       for model_name in ["bert-base-uncased"]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForSequenceClassification.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 def        a__				(       self      :						List[Any]						)     ->			Any:

       """simple docstring"""






       for model_name in ["bert-base-uncased"]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForQuestionAnswering.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 @slow
 @require_tensorflow_probability
 def        a__				(       self      :						int						)     ->			Union[str, Any]:

       """simple docstring"""






       for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
             lowerCamelCase_	  =					AutoConfig.from_pretrained(A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

             lowerCamelCase_	  =					TFAutoModelForTableQuestionAnswering.from_pretrained(A_						)
             lowerCamelCase_  ,       lowerCamelCase_	  =					TFAutoModelForTableQuestionAnswering.from_pretrained(
                 A_		,							output_loading_info=A_						)
             self.assertIsNotNone(A_						)
             self.assertIsInstance(A_		,							A_						)

 def        a__				(       self      :						int						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModelWithLMHead.from_pretrained(A_						)
       self.assertIsInstance(A_		,							A_						)
       self.assertEqual(model.num_parameters()		,							14410						)
       self.assertEqual(model.num_parameters(only_trainable=A_						)		,							14410						)

 def        a__				(       self      :						Optional[int]						)     ->			List[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModelWithLMHead.from_pretrained(A_						)
       self.assertIsInstance(A_		,							A_						)
       self.assertEqual(model.num_parameters()		,							14410						)
       self.assertEqual(model.num_parameters(only_trainable=A_						)		,							14410						)

 def        a__				(       self      :						Tuple						)     ->			Optional[Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModel.from_pretrained('sgugger/funnel-random-tiny'						)
       self.assertIsInstance(A_		,							A_						)

       lowerCamelCase_	  =					copy.deepcopy(model.config						)
       lowerCamelCase_	  =					['FunnelBaseModel']
       lowerCamelCase_	  =					TFAutoModel.from_config(A_						)
       self.assertIsInstance(A_		,							A_						)

       with tempfile.TemporaryDirectory() as tmp_dir:
             model.save_pretrained(A_						)
             lowerCamelCase_	  =					TFAutoModel.from_pretrained(A_						)
             self.assertIsInstance(A_		,							A_						)

 def        a__				(       self      :						Any						)     ->			Tuple:

       """simple docstring"""






       try:
             AutoConfig.register('new-model'		,							A_						)

             lowerCamelCase_	  =					[
                 TFAutoModel,
                 TFAutoModelForCausalLM,
                 TFAutoModelForMaskedLM,
                 TFAutoModelForPreTraining,
                 TFAutoModelForQuestionAnswering,
                 TFAutoModelForSequenceClassification,
                 TFAutoModelForTokenClassification,
             ]

             for auto_class in auto_classes:
                   with self.subTest(auto_class.__name__						):
                         # Wrong config class will raise an error
                         with self.assertRaises(A_						):
                               auto_class.register(A_		,							A_						)
                         auto_class.register(A_		,							A_						)
                         # Trying to register something existing in the Transformers library will raise an error
                         with self.assertRaises(A_						):
                               auto_class.register(A_		,							A_						)

                         # Now that the config is registered, it can be used as any other config with the auto-API
                         lowerCamelCase_	  =					BertModelTester(self						).get_config()
                         lowerCamelCase_	  =					NewModelConfig(**tiny_config.to_dict()						)
                         lowerCamelCase_	  =					auto_class.from_config(A_						)
                         self.assertIsInstance(A_		,							A_						)

                         with tempfile.TemporaryDirectory() as tmp_dir:
                               model.save_pretrained(A_						)
                               lowerCamelCase_	  =					auto_class.from_pretrained(A_						)
                               self.assertIsInstance(A_		,							A_						)

       finally:
             if "new-model" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["new-model"]
             for mapping in (
                 TF_MODEL_MAPPING,
                 TF_MODEL_FOR_PRETRAINING_MAPPING,
                 TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                 TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                 TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                 TF_MODEL_FOR_CAUSAL_LM_MAPPING,
                 TF_MODEL_FOR_MASKED_LM_MAPPING,
             ):
                   if NewModelConfig in mapping._extra_content:
                         del mapping._extra_content[NewModelConfig]

 def        a__				(       self      :						int						)     ->			int:

       """simple docstring"""






       with self.assertRaisesRegex(
           A_		,							'bert-base is not a local folder and is not a valid model identifier'						):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('bert-base'						)

 def        a__				(       self      :						Any						)     ->			Dict:

       """simple docstring"""






       with self.assertRaisesRegex(
           A_		,							r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'						):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained(A_		,							revision='aaaaaa'						)

 def        a__				(       self      :						str						)     ->			int:

       """simple docstring"""






       with self.assertRaisesRegex(
           A_		,							'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'		,							):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/config-no-model'						)

 def        a__				(       self      :						Any						)     ->			List[Any]:

       """simple docstring"""






       with self.assertRaisesRegex(A_		,							'Use `from_pt=True` to load this model'						):
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'						)





 def        a__				(       self      :						Tuple						)     ->			Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'						)
       with RequestCounter() as counter:
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'						)
             self.assertEqual(counter.get_request_count		,							0						)
             self.assertEqual(counter.head_request_count		,							1						)
             self.assertEqual(counter.other_request_count		,							0						)

       # With a sharded checkpoint
       lowerCamelCase_	  =					TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'						)
       with RequestCounter() as counter:
             lowerCamelCase_	  =					TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'						)
             self.assertEqual(counter.get_request_count		,							0						)
             self.assertEqual(counter.head_request_count		,							1						)
             self.assertEqual(counter.other_request_count		,							0						)



