



import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


UpperCAmelCase_									=							threading.Lock()
UpperCAmelCase_									=							None

UpperCAmelCase_									=							{
    "debug": logging.DEBUG,
    "info": logging.INFO,
    "warning": logging.WARNING,
    "error": logging.ERROR,
    "critical": logging.CRITICAL,
}

UpperCAmelCase_									=							logging.WARNING

UpperCAmelCase_									=							True
def 							UpperCAmelCase__		(				)->str:
      _lowerCAmelCase 			=  os.getenv('''TRANSFORMERS_VERBOSITY'''     ,				_SCREAMING_SNAKE_CASE				)
      if env_level_str:
            if env_level_str in log_levels:
                  return log_levels[env_level_str]
            else:
                  logging.getLogger().warning(
                      f'''Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, '''
                      f'''has to be one of: { ", ".join(log_levels.keys()				) }'''				)
      return _default_log_level
def 							UpperCAmelCase__		(				)->str:
      return __name__.split('''.'''				)[0]
def 							UpperCAmelCase__		(				)->logging.Logger:
      return logging.getLogger(_get_library_name()				)
def 							UpperCAmelCase__		(				)->None:
      global _default_handler

      with _lock:
            if _default_handler:
                  # This library has already configured the library root logger.
                  return
            _lowerCAmelCase 			=  logging.StreamHandler()  # Set sys.stderr as stream.
            _lowerCAmelCase 			=  sys.stderr.flush

            # Apply our default configuration to the library root logger.
            _lowerCAmelCase 			=  _get_library_root_logger()
            library_root_logger.addHandler(_default_handler				)
            library_root_logger.setLevel(_get_default_logging_level()				)
            _lowerCAmelCase 			=  False
def 							UpperCAmelCase__		(				)->None:
      global _default_handler

      with _lock:
            if not _default_handler:
                  return

            _lowerCAmelCase 			=  _get_library_root_logger()
            library_root_logger.removeHandler(_default_handler				)
            library_root_logger.setLevel(logging.NOTSET				)
            _lowerCAmelCase 			=  None
def 							UpperCAmelCase__		(				)->Optional[Any]:
      return log_levels
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Optional[str] = None				)->logging.Logger:

      if name is None:
            _lowerCAmelCase 			=  _get_library_name()

      _configure_library_root_logger()
      return logging.getLogger(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				)->int:
      _configure_library_root_logger()
      return _get_library_root_logger().getEffectiveLevel()
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	int				)->None:
      _configure_library_root_logger()
      _get_library_root_logger().setLevel(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				)->Optional[int]:
      return set_verbosity(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				)->Union[str, Any]:
      return set_verbosity(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				)->Dict:
      return set_verbosity(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				)->Optional[Any]:
      return set_verbosity(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				)->None:
      _configure_library_root_logger()

      assert _default_handler is not None
      _get_library_root_logger().removeHandler(_default_handler				)
def 							UpperCAmelCase__		(				)->None:
      _configure_library_root_logger()

      assert _default_handler is not None
      _get_library_root_logger().addHandler(_default_handler				)
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	logging.Handler				)->None:
      _configure_library_root_logger()

      assert handler is not None
      _get_library_root_logger().addHandler(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	logging.Handler				)->None:
      _configure_library_root_logger()

      assert handler is not None and handler not in _get_library_root_logger().handlers
      _get_library_root_logger().removeHandler(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				)->None:
      _configure_library_root_logger()
      _lowerCAmelCase 			=  False
def 							UpperCAmelCase__		(				)->None:
      _configure_library_root_logger()
      _lowerCAmelCase 			=  True
def 							UpperCAmelCase__		(				)->None:
      _lowerCAmelCase 			=  _get_library_root_logger().handlers

      for handler in handlers:
            _lowerCAmelCase 			=  logging.Formatter('''[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s'''				)
            handler.setFormatter(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				)->None:
      _lowerCAmelCase 			=  _get_library_root_logger().handlers

      for handler in handlers:
            handler.setFormatter(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				self   :	Any     ,				*_SCREAMING_SNAKE_CASE   :	List[str]     ,				**_SCREAMING_SNAKE_CASE   :	Union[str, Any]				)->Optional[Any]:
      _lowerCAmelCase 			=  os.getenv('''TRANSFORMERS_NO_ADVISORY_WARNINGS'''     ,				_SCREAMING_SNAKE_CASE				)
      if no_advisory_warnings:
            return
      self.warning(*_SCREAMING_SNAKE_CASE     ,				**_SCREAMING_SNAKE_CASE				)


UpperCAmelCase_									=							warning_advice
@functools.lru_cache(_SCREAMING_SNAKE_CASE				)
def 							UpperCAmelCase__		(				self   :	List[Any]     ,				*_SCREAMING_SNAKE_CASE   :	List[str]     ,				**_SCREAMING_SNAKE_CASE   :	Optional[int]				)->Dict:
      self.warning(*_SCREAMING_SNAKE_CASE     ,				**_SCREAMING_SNAKE_CASE				)


UpperCAmelCase_									=							warning_once





class 		UpperCAmelCase						:





   def __init__(	self       ,     *_lowerCAmelCase       ,     **_lowerCAmelCase			):  # pylint: disable=unused-argument
         _lowerCAmelCase 			=  args[0] if args else None





   def __iter__(	self			):
         return iter(self._iterator			)





   def __getattr__(	self       ,     _lowerCAmelCase			):

         def empty_fn(*_lowerCAmelCase       ,     **_lowerCAmelCase			):  # pylint: disable=unused-argument
               return

         return empty_fn





   def __enter__(	self			):
         return self





   def __exit__(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         return






class 		UpperCAmelCase						:





   def __call__(	self       ,     *_lowerCAmelCase       ,     **_lowerCAmelCase			):
         if _tqdm_active:
               return tqdm_lib.tqdm(*_lowerCAmelCase       ,     **_lowerCAmelCase			)
         else:
               return EmptyTqdm(*_lowerCAmelCase       ,     **_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self       ,     *_lowerCAmelCase       ,     **_lowerCAmelCase			):
         _lowerCAmelCase 			=  None
         if _tqdm_active:
               return tqdm_lib.tqdm.set_lock(*_lowerCAmelCase       ,     **_lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         if _tqdm_active:
               return tqdm_lib.tqdm.get_lock()






UpperCAmelCase_									=							_tqdm_cls()
def 							UpperCAmelCase__		(				)->bool:
      global _tqdm_active
      return bool(_tqdm_active				)
def 							UpperCAmelCase__		(				)->List[Any]:
      global _tqdm_active
      _lowerCAmelCase 			=  True
      hf_hub_utils.enable_progress_bars()


def 							UpperCAmelCase__		(				)->Optional[int]:
      global _tqdm_active
      _lowerCAmelCase 			=  False
      hf_hub_utils.disable_progress_bars()




import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("1.0.0a"):
    raise Exception("requires fairseq >= 1.0.0a")

logging.set_verbosity_info()
UpperCAmelCase_									=							logging.get_logger(__name__)

UpperCAmelCase_									=							"Hello world! cécé herlolip"
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	str     ,				_SCREAMING_SNAKE_CASE   :	str     ,				_SCREAMING_SNAKE_CASE   :	bool				)->List[Any]:
      _lowerCAmelCase 			=  FairseqRobertaModel.from_pretrained(_SCREAMING_SNAKE_CASE				)
      roberta.eval()  # disable dropout
      _lowerCAmelCase 			=  roberta.model.encoder.sentence_encoder
      _lowerCAmelCase 			=  XLMRobertaConfig(
          vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings     ,				hidden_size=roberta.cfg.model.encoder_embed_dim     ,				num_hidden_layers=roberta.cfg.model.encoder_layers     ,				num_attention_heads=roberta.cfg.model.encoder_attention_heads     ,				intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim     ,				max_position_embeddings=5_1_4     ,				type_vocab_size=1     ,				layer_norm_eps=1e-5     ,				)
      if classification_head:
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''].out_proj.weight.shape[0]

      print('''Our RoBERTa config:'''     ,				_SCREAMING_SNAKE_CASE				)

      _lowerCAmelCase 			=  XLMRobertaXLForSequenceClassification(_SCREAMING_SNAKE_CASE				) if classification_head else XLMRobertaXLForMaskedLM(_SCREAMING_SNAKE_CASE				)
      model.eval()

      # Now let's copy all the weights.
      # Embeddings
      _lowerCAmelCase 			=  roberta_sent_encoder.embed_tokens.weight
      _lowerCAmelCase 			=  roberta_sent_encoder.embed_positions.weight
      _lowerCAmelCase 			=  torch.zeros_like(
          model.roberta.embeddings.token_type_embeddings.weight				)  # just zero them out b/c RoBERTa doesn't use them.

      _lowerCAmelCase 			=  roberta_sent_encoder.layer_norm.weight
      _lowerCAmelCase 			=  roberta_sent_encoder.layer_norm.bias

      for i in range(config.num_hidden_layers				):
            # Encoder: start of layer
            _lowerCAmelCase 			=  model.roberta.encoder.layer[i]
            _lowerCAmelCase 			=  roberta_sent_encoder.layers[i]

            _lowerCAmelCase 			=  layer.attention
            _lowerCAmelCase 			=  roberta_layer.self_attn_layer_norm.weight
            _lowerCAmelCase 			=  roberta_layer.self_attn_layer_norm.bias

            # self attention
            _lowerCAmelCase 			=  layer.attention.self
            assert (
                roberta_layer.self_attn.k_proj.weight.data.shape
                == roberta_layer.self_attn.q_proj.weight.data.shape
                == roberta_layer.self_attn.v_proj.weight.data.shape
                == torch.Size((config.hidden_size, config.hidden_size)				)
            )

            _lowerCAmelCase 			=  roberta_layer.self_attn.q_proj.weight
            _lowerCAmelCase 			=  roberta_layer.self_attn.q_proj.bias
            _lowerCAmelCase 			=  roberta_layer.self_attn.k_proj.weight
            _lowerCAmelCase 			=  roberta_layer.self_attn.k_proj.bias
            _lowerCAmelCase 			=  roberta_layer.self_attn.v_proj.weight
            _lowerCAmelCase 			=  roberta_layer.self_attn.v_proj.bias

            # self-attention output
            _lowerCAmelCase 			=  layer.attention.output
            assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
            _lowerCAmelCase 			=  roberta_layer.self_attn.out_proj.weight
            _lowerCAmelCase 			=  roberta_layer.self_attn.out_proj.bias

            # this one is final layer norm
            _lowerCAmelCase 			=  roberta_layer.final_layer_norm.weight
            _lowerCAmelCase 			=  roberta_layer.final_layer_norm.bias

            # intermediate
            _lowerCAmelCase 			=  layer.intermediate
            assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
            _lowerCAmelCase 			=  roberta_layer.fca.weight
            _lowerCAmelCase 			=  roberta_layer.fca.bias

            # output
            _lowerCAmelCase 			=  layer.output
            assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
            _lowerCAmelCase 			=  roberta_layer.fca.weight
            _lowerCAmelCase 			=  roberta_layer.fca.bias
            # end of layer

      if classification_head:
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''].dense.weight
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''].dense.bias
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''].out_proj.weight
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''].out_proj.bias
      else:
            # LM Head
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.dense.weight
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.dense.bias
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.layer_norm.weight
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.layer_norm.bias
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.weight
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.bias

      # Let's check that we get the same results.
      _lowerCAmelCase 			=  roberta.encode(_SCREAMING_SNAKE_CASE				).unsqueeze(0				)  # batch of size 1

      _lowerCAmelCase 			=  model(_SCREAMING_SNAKE_CASE				)[0]
      if classification_head:
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''](roberta.extract_features(_SCREAMING_SNAKE_CASE				)				)
      else:
            _lowerCAmelCase 			=  roberta.model(_SCREAMING_SNAKE_CASE				)[0]
      print(our_output.shape     ,				their_output.shape				)
      _lowerCAmelCase 			=  torch.max(torch.abs(our_output - their_output				)				).item()
      print(f'''max_absolute_diff = {max_absolute_diff}'''				)  # ~ 1e-7
      _lowerCAmelCase 			=  torch.allclose(_SCREAMING_SNAKE_CASE     ,				_SCREAMING_SNAKE_CASE     ,				atol=1e-3				)
      print('''Do both models output the same tensors?'''     ,				'''🔥''' if success else '''💩'''				)
      if not success:
            raise Exception('''Something went wRoNg'''				)

      pathlib.Path(_SCREAMING_SNAKE_CASE				).mkdir(parents=_SCREAMING_SNAKE_CASE     ,				exist_ok=_SCREAMING_SNAKE_CASE				)
      print(f'''Saving model to {pytorch_dump_folder_path}'''				)
      model.save_pretrained(_SCREAMING_SNAKE_CASE				)


if __name__ == "__main__":
    UpperCAmelCase_									=							argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        "--roberta_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
    )
    parser.add_argument(
        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
    )
    parser.add_argument(
        "--classification_head", action="store_true", help="Whether to convert a final classification head."
    )
    UpperCAmelCase_									=							parser.parse_args()
    convert_xlm_roberta_xl_checkpoint_to_pytorch(
        args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
    )