



import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


UpperCAmelCase_									=							datasets.utils.logging.get_logger(__name__)





@dataclass
class 		UpperCAmelCase						(						datasets.BuilderConfig				):
   SCREAMING_SNAKE_CASE__							=     None
   SCREAMING_SNAKE_CASE__							=     '''utf-8'''
   SCREAMING_SNAKE_CASE__							=     None
   SCREAMING_SNAKE_CASE__							=     None
   SCREAMING_SNAKE_CASE__							=     True  # deprecated
   SCREAMING_SNAKE_CASE__							=     None  # deprecated
   SCREAMING_SNAKE_CASE__							=     1_0 << 2_0  # 10MB
   SCREAMING_SNAKE_CASE__							=     None






class 		UpperCAmelCase						(						datasets.ArrowBasedBuilder				):
   SCREAMING_SNAKE_CASE__							=     JsonConfig





   def 	__lowerCAmelCase							(	self			):
          if self.config.block_size is not None:
                 logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''			)
                 _lowerCAmelCase 			=  self.config.block_size
          if self.config.use_threads is not True:
                 logger.warning(
                     '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''			)
          if self.config.newlines_in_values is not None:
                 raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''			)
          return datasets.DatasetInfo(features=self.config.features			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase			):
          if not self.config.data_files:
                 raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''			)
          _lowerCAmelCase 			=  dl_manager.download_and_extract(self.config.data_files			)
          if isinstance(_lowerCAmelCase       ,     (str, list, tuple)			):
                 _lowerCAmelCase 			=  data_files
                 if isinstance(_lowerCAmelCase       ,     _lowerCAmelCase			):
                        _lowerCAmelCase 			=  [files]
                 _lowerCAmelCase 			=  [dl_manager.iter_files(_lowerCAmelCase			) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,     gen_kwargs={'''files''': files}			)]
          _lowerCAmelCase 			=  []
          for split_name, files in data_files.items():
                 if isinstance(_lowerCAmelCase       ,     _lowerCAmelCase			):
                        _lowerCAmelCase 			=  [files]
                 _lowerCAmelCase 			=  [dl_manager.iter_files(_lowerCAmelCase			) for file in files]
                 splits.append(datasets.SplitGenerator(name=_lowerCAmelCase       ,     gen_kwargs={'''files''': files}			)			)
          return splits





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase			):
          if self.config.features is not None:
                 # adding missing columns
                 for column_name in set(self.config.features			) - set(pa_table.column_names			):
                        _lowerCAmelCase 			=  self.config.features.arrow_schema.field(_lowerCAmelCase			).type
                        _lowerCAmelCase 			=  pa_table.append_column(_lowerCAmelCase       ,     pa.array([None] * len(_lowerCAmelCase			)       ,     type=_lowerCAmelCase			)			)
                 # more expensive cast to support nested structures with keys in a different order
                 # allows str <-> int/float or str to Audio for example
                 _lowerCAmelCase 			=  table_cast(_lowerCAmelCase       ,     self.config.features.arrow_schema			)
          return pa_table





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase			):
          for file_idx, file in enumerate(itertools.chain.from_iterable(_lowerCAmelCase			)			):
                 # If the file is one json object and if we need to look at the list of items in one specific field
                 if self.config.field is not None:
                        with open(_lowerCAmelCase       ,     encoding=self.config.encoding       ,     errors=self.config.encoding_errors			) as f:
                               _lowerCAmelCase 			=  json.load(_lowerCAmelCase			)

                        # We keep only the field we are interested in
                        _lowerCAmelCase 			=  dataset[self.config.field]

                        # We accept two format: a list of dicts or a dict of lists
                        if isinstance(_lowerCAmelCase       ,     (list, tuple)			):
                               _lowerCAmelCase 			=  set().union(*[row.keys() for row in dataset]			)
                               _lowerCAmelCase 			=  {col: [row.get(_lowerCAmelCase			) for row in dataset] for col in keys}
                        else:
                               _lowerCAmelCase 			=  dataset
                        _lowerCAmelCase 			=  pa.Table.from_pydict(_lowerCAmelCase			)
                        yield file_idx, self._cast_table(_lowerCAmelCase			)

                 # If the file has one json object per line
                 else:
                        with open(_lowerCAmelCase       ,     '''rb'''			) as f:
                               _lowerCAmelCase 			=  0
                               # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                               # Set a default minimum value of 16kB if the chunk size is really small
                               _lowerCAmelCase 			=  max(self.config.chunksize // 32       ,     16 << 10			)
                               _lowerCAmelCase 			=  (
                                   self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
                               )
                               while True:
                                      _lowerCAmelCase 			=  f.read(self.config.chunksize			)
                                      if not batch:
                                             break
                                      # Finish current line
                                      try:
                                             batch += f.readline()
                                      except (AttributeError, io.UnsupportedOperation):
                                             batch += readline(_lowerCAmelCase			)
                                      # PyArrow only accepts utf-8 encoded bytes
                                      if self.config.encoding != "utf-8":
                                             _lowerCAmelCase 			=  batch.decode(self.config.encoding       ,     errors=_lowerCAmelCase			).encode('''utf-8'''			)
                                      try:
                                             while True:
                                                    try:
                                                           _lowerCAmelCase 			=  paj.read_json(
                                                               io.BytesIO(_lowerCAmelCase			)       ,     read_options=paj.ReadOptions(block_size=_lowerCAmelCase			)			)
                                                           break
                                                    except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                           if (
                                                               isinstance(_lowerCAmelCase       ,     pa.ArrowInvalid			)
                                                               and "straddling" not in str(_lowerCAmelCase			)
                                                               or block_size > len(_lowerCAmelCase			)
                                                           ):
                                                                  raise
                                                           else:
                                                                  # Increase the block size in case it was too small.
                                                                  # The block size will be reset for the next file.
                                                                  logger.debug(
                                                                      F'''Batch of {len(_lowerCAmelCase			)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''			)
                                                                  block_size *= 2
                                      except pa.ArrowInvalid as e:
                                             try:
                                                    with open(
                                                        _lowerCAmelCase       ,     encoding=self.config.encoding       ,     errors=self.config.encoding_errors			) as f:
                                                           _lowerCAmelCase 			=  json.load(_lowerCAmelCase			)
                                             except json.JSONDecodeError:
                                                    logger.error(F'''Failed to read file \'{file}\' with error {type(_lowerCAmelCase			)}: {e}'''			)
                                                    raise e
                                             # If possible, parse the file as a list of json objects and exit the loop
                                             if isinstance(_lowerCAmelCase       ,     _lowerCAmelCase			):  # list is the only sequence type supported in JSON
                                                    try:
                                                           _lowerCAmelCase 			=  set().union(*[row.keys() for row in dataset]			)
                                                           _lowerCAmelCase 			=  {col: [row.get(_lowerCAmelCase			) for row in dataset] for col in keys}
                                                           _lowerCAmelCase 			=  pa.Table.from_pydict(_lowerCAmelCase			)
                                                    except (pa.ArrowInvalid, AttributeError) as e:
                                                           logger.error(F'''Failed to read file \'{file}\' with error {type(_lowerCAmelCase			)}: {e}'''			)
                                                           raise ValueError(F'''Not able to read records in the JSON file at {file}.'''			) from None
                                                    yield file_idx, self._cast_table(_lowerCAmelCase			)
                                                    break
                                             else:
                                                    logger.error(F'''Failed to read file \'{file}\' with error {type(_lowerCAmelCase			)}: {e}'''			)
                                                    raise ValueError(
                                                        F'''Not able to read records in the JSON file at {file}. '''
                                                        F'''You should probably indicate the field of the JSON file containing your records. '''
                                                        F'''This JSON file contain the following fields: {str(list(dataset.keys()			)			)}. '''
                                                        F'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''			) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                      yield (file_idx, batch_idx), self._cast_table(_lowerCAmelCase			)
                                      batch_idx += 1




import argparse
import pathlib

import fairseq
import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from packaging import version

from transformers import XLMRobertaConfig, XLMRobertaXLForMaskedLM, XLMRobertaXLForSequenceClassification
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.models.roberta.modeling_roberta import RobertaAttention
from transformers.utils import logging


if version.parse(fairseq.__version__) < version.parse("1.0.0a"):
    raise Exception("requires fairseq >= 1.0.0a")

logging.set_verbosity_info()
UpperCAmelCase_									=							logging.get_logger(__name__)

UpperCAmelCase_									=							"Hello world! cécé herlolip"
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	str     ,				_SCREAMING_SNAKE_CASE   :	str     ,				_SCREAMING_SNAKE_CASE   :	bool				)->List[Any]:
      _lowerCAmelCase 			=  FairseqRobertaModel.from_pretrained(_SCREAMING_SNAKE_CASE				)
      roberta.eval()  # disable dropout
      _lowerCAmelCase 			=  roberta.model.encoder.sentence_encoder
      _lowerCAmelCase 			=  XLMRobertaConfig(
          vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings     ,				hidden_size=roberta.cfg.model.encoder_embed_dim     ,				num_hidden_layers=roberta.cfg.model.encoder_layers     ,				num_attention_heads=roberta.cfg.model.encoder_attention_heads     ,				intermediate_size=roberta.cfg.model.encoder_ffn_embed_dim     ,				max_position_embeddings=5_1_4     ,				type_vocab_size=1     ,				layer_norm_eps=1e-5     ,				)
      if classification_head:
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''].out_proj.weight.shape[0]

      print('''Our RoBERTa config:'''     ,				_SCREAMING_SNAKE_CASE				)

      _lowerCAmelCase 			=  XLMRobertaXLForSequenceClassification(_SCREAMING_SNAKE_CASE				) if classification_head else XLMRobertaXLForMaskedLM(_SCREAMING_SNAKE_CASE				)
      model.eval()

      # Now let's copy all the weights.
      # Embeddings
      _lowerCAmelCase 			=  roberta_sent_encoder.embed_tokens.weight
      _lowerCAmelCase 			=  roberta_sent_encoder.embed_positions.weight
      _lowerCAmelCase 			=  torch.zeros_like(
          model.roberta.embeddings.token_type_embeddings.weight				)  # just zero them out b/c RoBERTa doesn't use them.

      _lowerCAmelCase 			=  roberta_sent_encoder.layer_norm.weight
      _lowerCAmelCase 			=  roberta_sent_encoder.layer_norm.bias

      for i in range(config.num_hidden_layers				):
            # Encoder: start of layer
            _lowerCAmelCase 			=  model.roberta.encoder.layer[i]
            _lowerCAmelCase 			=  roberta_sent_encoder.layers[i]

            _lowerCAmelCase 			=  layer.attention
            _lowerCAmelCase 			=  roberta_layer.self_attn_layer_norm.weight
            _lowerCAmelCase 			=  roberta_layer.self_attn_layer_norm.bias

            # self attention
            _lowerCAmelCase 			=  layer.attention.self
            assert (
                roberta_layer.self_attn.k_proj.weight.data.shape
                == roberta_layer.self_attn.q_proj.weight.data.shape
                == roberta_layer.self_attn.v_proj.weight.data.shape
                == torch.Size((config.hidden_size, config.hidden_size)				)
            )

            _lowerCAmelCase 			=  roberta_layer.self_attn.q_proj.weight
            _lowerCAmelCase 			=  roberta_layer.self_attn.q_proj.bias
            _lowerCAmelCase 			=  roberta_layer.self_attn.k_proj.weight
            _lowerCAmelCase 			=  roberta_layer.self_attn.k_proj.bias
            _lowerCAmelCase 			=  roberta_layer.self_attn.v_proj.weight
            _lowerCAmelCase 			=  roberta_layer.self_attn.v_proj.bias

            # self-attention output
            _lowerCAmelCase 			=  layer.attention.output
            assert self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape
            _lowerCAmelCase 			=  roberta_layer.self_attn.out_proj.weight
            _lowerCAmelCase 			=  roberta_layer.self_attn.out_proj.bias

            # this one is final layer norm
            _lowerCAmelCase 			=  roberta_layer.final_layer_norm.weight
            _lowerCAmelCase 			=  roberta_layer.final_layer_norm.bias

            # intermediate
            _lowerCAmelCase 			=  layer.intermediate
            assert intermediate.dense.weight.shape == roberta_layer.fca.weight.shape
            _lowerCAmelCase 			=  roberta_layer.fca.weight
            _lowerCAmelCase 			=  roberta_layer.fca.bias

            # output
            _lowerCAmelCase 			=  layer.output
            assert bert_output.dense.weight.shape == roberta_layer.fca.weight.shape
            _lowerCAmelCase 			=  roberta_layer.fca.weight
            _lowerCAmelCase 			=  roberta_layer.fca.bias
            # end of layer

      if classification_head:
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''].dense.weight
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''].dense.bias
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''].out_proj.weight
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''].out_proj.bias
      else:
            # LM Head
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.dense.weight
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.dense.bias
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.layer_norm.weight
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.layer_norm.bias
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.weight
            _lowerCAmelCase 			=  roberta.model.encoder.lm_head.bias

      # Let's check that we get the same results.
      _lowerCAmelCase 			=  roberta.encode(_SCREAMING_SNAKE_CASE				).unsqueeze(0				)  # batch of size 1

      _lowerCAmelCase 			=  model(_SCREAMING_SNAKE_CASE				)[0]
      if classification_head:
            _lowerCAmelCase 			=  roberta.model.classification_heads['''mnli'''](roberta.extract_features(_SCREAMING_SNAKE_CASE				)				)
      else:
            _lowerCAmelCase 			=  roberta.model(_SCREAMING_SNAKE_CASE				)[0]
      print(our_output.shape     ,				their_output.shape				)
      _lowerCAmelCase 			=  torch.max(torch.abs(our_output - their_output				)				).item()
      print(f'''max_absolute_diff = {max_absolute_diff}'''				)  # ~ 1e-7
      _lowerCAmelCase 			=  torch.allclose(_SCREAMING_SNAKE_CASE     ,				_SCREAMING_SNAKE_CASE     ,				atol=1e-3				)
      print('''Do both models output the same tensors?'''     ,				'''🔥''' if success else '''💩'''				)
      if not success:
            raise Exception('''Something went wRoNg'''				)

      pathlib.Path(_SCREAMING_SNAKE_CASE				).mkdir(parents=_SCREAMING_SNAKE_CASE     ,				exist_ok=_SCREAMING_SNAKE_CASE				)
      print(f'''Saving model to {pytorch_dump_folder_path}'''				)
      model.save_pretrained(_SCREAMING_SNAKE_CASE				)


if __name__ == "__main__":
    UpperCAmelCase_									=							argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        "--roberta_checkpoint_path", default=None, type=str, required=True, help="Path the official PyTorch dump."
    )
    parser.add_argument(
        "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
    )
    parser.add_argument(
        "--classification_head", action="store_true", help="Whether to convert a final classification head."
    )
    UpperCAmelCase_									=							parser.parse_args()
    convert_xlm_roberta_xl_checkpoint_to_pytorch(
        args.roberta_checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
    )