






import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse('0.8.3'):
							raise Exception('requires gluonnlp == 0.8.3')

if version.parse(mx.__version__) != version.parse('1.5.0'):
							raise Exception('requires mxnet == 1.5.0')

logging.set_verbosity_info()
A__			:    int         =				logging.get_logger(__name__)

A__			:    List[Any]         =				'The Nymphenburg Palace is a beautiful palace in Munich!'






def 						a    (		lowerCamelCase_	,     lowerCamelCase_       ):






				'''simple docstring'''
				lowercase__     				=    {
				    '''attention_cell''': '''multi_head''',
				    '''num_layers''': 4,
				    '''units''': 1024,
				    '''hidden_size''': 768,
				    '''max_length''': 512,
				    '''num_heads''': 8,
				    '''scaled''': True,
				    '''dropout''': 0.1,
				    '''use_residual''': True,
				    '''embed_size''': 1024,
				    '''embed_dropout''': 0.1,
				    '''word_embed''': None,
				    '''layer_norm_eps''': 1e-5,
				    '''token_type_vocab_size''': 2,
				}

				lowercase__     				=    bort_4_8_768_1024_hparams

				# Let's construct the original Bort model here
				# Taken from official BERT implementation, see:
				# https://github.com/alexa/bort/blob/master/bort/bort.py
				lowercase__     				=    BERTEncoder(
				    attention_cell=predefined_args['''attention_cell''']	,     num_layers=predefined_args['''num_layers''']	,     units=predefined_args['''units''']	,     hidden_size=predefined_args['''hidden_size''']	,     max_length=predefined_args['''max_length''']	,     num_heads=predefined_args['''num_heads''']	,     scaled=predefined_args['''scaled''']	,     dropout=predefined_args['''dropout''']	,     output_attention=lowerCamelCase_	,     output_all_encodings=lowerCamelCase_	,     use_residual=predefined_args['''use_residual''']	,     activation=predefined_args.get('''activation'''	,     '''gelu'''       )	,     layer_norm_eps=predefined_args.get('''layer_norm_eps'''	,     lowerCamelCase_       )	,     )

				# Vocab information needs to be fetched first
				# It's the same as RoBERTa, so RobertaTokenizer can be used later
				lowercase__     				=    '''openwebtext_ccnews_stories_books_cased'''

				# Specify download folder to Gluonnlp's vocab
				lowercase__     				=    os.path.join(get_home_dir()	,     '''models'''       )
				lowercase__     				=    _load_vocab(lowerCamelCase_	,     lowerCamelCase_	,     lowerCamelCase_	,     cls=lowerCamelCase_       )

				lowercase__     				=    nlp.model.BERTModel(
				    lowerCamelCase_	,     len(lowerCamelCase_       )	,     units=predefined_args['''units''']	,     embed_size=predefined_args['''embed_size''']	,     embed_dropout=predefined_args['''embed_dropout''']	,     word_embed=predefined_args['''word_embed''']	,     use_pooler=lowerCamelCase_	,     use_token_type_embed=lowerCamelCase_	,     token_type_vocab_size=predefined_args['''token_type_vocab_size''']	,     use_classifier=lowerCamelCase_	,     use_decoder=lowerCamelCase_	,     )

				original_bort.load_parameters(lowerCamelCase_	,     cast_dtype=lowerCamelCase_	,     ignore_extra=lowerCamelCase_       )
				lowercase__     				=    original_bort._collect_params_with_prefix()

				# Build our config 🤗
				lowercase__     				=    {
				    '''architectures''': ['''BertForMaskedLM'''],
				    '''attention_probs_dropout_prob''': predefined_args['''dropout'''],
				    '''hidden_act''': '''gelu''',
				    '''hidden_dropout_prob''': predefined_args['''dropout'''],
				    '''hidden_size''': predefined_args['''embed_size'''],
				    '''initializer_range''': 0.02,
				    '''intermediate_size''': predefined_args['''hidden_size'''],
				    '''layer_norm_eps''': predefined_args['''layer_norm_eps'''],
				    '''max_position_embeddings''': predefined_args['''max_length'''],
				    '''model_type''': '''bort''',
				    '''num_attention_heads''': predefined_args['''num_heads'''],
				    '''num_hidden_layers''': predefined_args['''num_layers'''],
				    '''pad_token_id''': 1,  # 2 = BERT, 1 = RoBERTa
				    '''type_vocab_size''': 1,  # 2 = BERT, 1 = RoBERTa
				    '''vocab_size''': len(lowerCamelCase_       ),
				}

				lowercase__     				=    BertConfig.from_dict(lowerCamelCase_       )
				lowercase__     				=    BertForMaskedLM(lowerCamelCase_       )
				hf_bort_model.eval()

				# Parameter mapping table (Gluonnlp to Transformers)
				# * denotes layer index
				#
				# | Gluon Parameter                                                | Transformers Parameter
				# | -------------------------------------------------------------- | ----------------------
				# | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
				# | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
				# | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
				# | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
				# | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
				# | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
				# | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
				# | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
				# | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
				# | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
				# | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
				# | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
				# | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
				# | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
				# | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
				# | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
				# | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
				# | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
				# | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
				# | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

				# Helper function to convert MXNET Arrays to PyTorch
				def to_torch(lowerCamelCase_       ) -> nn.Parameter:
								return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()       )       )

				# Check param shapes and map new HF param back
				def check_and_map_params(lowerCamelCase_	,     lowerCamelCase_       ):
								lowercase__     				=    hf_param.shape

								lowercase__     				=    to_torch(params[gluon_param]       )
								lowercase__     				=    gluon_param.shape

								assert (
								    shape_hf == shape_gluon
								), F"""The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers"""

								return gluon_param

				lowercase__     				=    check_and_map_params(
				    hf_bort_model.bert.embeddings.word_embeddings.weight	,     '''word_embed.0.weight'''       )
				lowercase__     				=    check_and_map_params(
				    hf_bort_model.bert.embeddings.position_embeddings.weight	,     '''encoder.position_weight'''       )
				lowercase__     				=    check_and_map_params(
				    hf_bort_model.bert.embeddings.LayerNorm.bias	,     '''encoder.layer_norm.beta'''       )
				lowercase__     				=    check_and_map_params(
				    hf_bort_model.bert.embeddings.LayerNorm.weight	,     '''encoder.layer_norm.gamma'''       )

				# Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
				lowercase__     				=    torch.zeros_like(
				    hf_bort_model.bert.embeddings.token_type_embeddings.weight.data       )

				for i in range(hf_bort_config.num_hidden_layers       ):
								lowercase__     				=    hf_bort_model.bert.encoder.layer[i]

								# self attention
								lowercase__     				=    layer.attention.self

								lowercase__     				=    check_and_map_params(
								    self_attn.key.bias.data	,     F"""encoder.transformer_cells.{i}.attention_cell.proj_key.bias"""       )

								lowercase__     				=    check_and_map_params(
								    self_attn.key.weight.data	,     F"""encoder.transformer_cells.{i}.attention_cell.proj_key.weight"""       )
								lowercase__     				=    check_and_map_params(
								    self_attn.query.bias.data	,     F"""encoder.transformer_cells.{i}.attention_cell.proj_query.bias"""       )
								lowercase__     				=    check_and_map_params(
								    self_attn.query.weight.data	,     F"""encoder.transformer_cells.{i}.attention_cell.proj_query.weight"""       )
								lowercase__     				=    check_and_map_params(
								    self_attn.value.bias.data	,     F"""encoder.transformer_cells.{i}.attention_cell.proj_value.bias"""       )
								lowercase__     				=    check_and_map_params(
								    self_attn.value.weight.data	,     F"""encoder.transformer_cells.{i}.attention_cell.proj_value.weight"""       )

								# self attention output
								lowercase__     				=    layer.attention.output

								lowercase__     				=    check_and_map_params(
								    self_output.dense.bias	,     F"""encoder.transformer_cells.{i}.proj.bias"""       )
								lowercase__     				=    check_and_map_params(
								    self_output.dense.weight	,     F"""encoder.transformer_cells.{i}.proj.weight"""       )
								lowercase__     				=    check_and_map_params(
								    self_output.LayerNorm.bias	,     F"""encoder.transformer_cells.{i}.layer_norm.beta"""       )
								lowercase__     				=    check_and_map_params(
								    self_output.LayerNorm.weight	,     F"""encoder.transformer_cells.{i}.layer_norm.gamma"""       )

								# intermediate
								lowercase__     				=    layer.intermediate

								lowercase__     				=    check_and_map_params(
								    intermediate.dense.bias	,     F"""encoder.transformer_cells.{i}.ffn.ffn_1.bias"""       )
								lowercase__     				=    check_and_map_params(
								    intermediate.dense.weight	,     F"""encoder.transformer_cells.{i}.ffn.ffn_1.weight"""       )

								# output
								lowercase__     				=    layer.output

								lowercase__     				=    check_and_map_params(
								    bert_output.dense.bias	,     F"""encoder.transformer_cells.{i}.ffn.ffn_2.bias"""       )
								lowercase__     				=    check_and_map_params(
								    bert_output.dense.weight	,     F"""encoder.transformer_cells.{i}.ffn.ffn_2.weight"""       )
								lowercase__     				=    check_and_map_params(
								    bert_output.LayerNorm.bias	,     F"""encoder.transformer_cells.{i}.ffn.layer_norm.beta"""       )
								lowercase__     				=    check_and_map_params(
								    bert_output.LayerNorm.weight	,     F"""encoder.transformer_cells.{i}.ffn.layer_norm.gamma"""       )

				# Save space and energy 🎄
				hf_bort_model.half()

				# Compare output of both models
				lowercase__     				=    RobertaTokenizer.from_pretrained('''roberta-base'''       )

				lowercase__     				=    tokenizer.encode_plus(lowerCamelCase_       )['''input_ids''']

				# Get gluon output
				lowercase__     				=    mx.nd.array([input_ids]       )
				lowercase__     				=    original_bort(inputs=lowerCamelCase_	,     token_types=[]       )

				# Get Transformer output (save and reload model again)
				hf_bort_model.save_pretrained(lowerCamelCase_       )
				lowercase__     				=    BertModel.from_pretrained(lowerCamelCase_       )
				hf_bort_model.eval()

				lowercase__     				=    tokenizer.encode_plus(lowerCamelCase_	,     return_tensors='''pt'''       )
				lowercase__     				=    hf_bort_model(**lowerCamelCase_       )[0]

				lowercase__     				=    output_gluon[0].asnumpy()
				lowercase__     				=    output_hf[0].detach().numpy()

				lowercase__     				=    np.max(np.abs(hf_layer - gluon_layer       )       ).item()
				lowercase__     				=    np.allclose(lowerCamelCase_	,     lowerCamelCase_	,     atol=1e-3       )

				if success:
								print('''✔️ Both model do output the same tensors'''       )
				else:
								print('''❌ Both model do **NOT** output the same tensors'''       )
								print('''Absolute difference is:'''	,     lowerCamelCase_       )


if __name__ == "__main__":
							A__			:    List[str]         =				argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--bort_checkpoint_path', default=None, type=str, required=True, help='Path the official Bort params file.'
							)
							parser.add_argument(
							    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
							)
							A__			:    str         =				parser.parse_args()
							convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)












import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

if is_vision_available():
							from PIL import Image

							from transformers import MobileNetVaImageProcessor
class    _UpperCAmelCase      ( unittest.TestCase			):

				"""simple docstring"""




				def __init__(  self		:					str,						lowerCamelCase		:					Any,						lowerCamelCase		:					Tuple=7,						lowerCamelCase		:					str=3,						lowerCamelCase		:					Tuple=18,						lowerCamelCase		:					int=30,						lowerCamelCase		:					Tuple=400,						lowerCamelCase		:					Any=True,						lowerCamelCase		:					Any=None,						lowerCamelCase		:					List[str]=True,						lowerCamelCase		:					Union[str, Any]=None,						):

								'''simple docstring'''
								lowercase__     				=    size if size is not None else {'''shortest_edge''': 20}
								lowercase__     				=    crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}
								lowercase__     				=    parent
								lowercase__     				=    batch_size
								lowercase__     				=    num_channels
								lowercase__     				=    image_size
								lowercase__     				=    min_resolution
								lowercase__     				=    max_resolution
								lowercase__     				=    do_resize
								lowercase__     				=    size
								lowercase__     				=    do_center_crop
								lowercase__     				=    crop_size




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								return {
								    "do_resize": self.do_resize,
								    "size": self.size,
								    "do_center_crop": self.do_center_crop,
								    "crop_size": self.crop_size,
								}



@require_torch
@require_vision
class    _UpperCAmelCase      ( A__	,unittest.TestCase			):

				"""simple docstring"""




				lowercase__       							= MobileNetVaImageProcessor if is_vision_available() else None




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								lowercase__     				=    MobileNetVaImageProcessingTester(self						)




				@property
				def        lowercase__     (  self		:					Union[str, Any]						):

								'''simple docstring'''
								return self.image_processor_tester.prepare_image_processor_dict()




				def        lowercase__     (  self		:					Dict						):

								'''simple docstring'''
								lowercase__     				=    self.image_processing_class(**self.image_processor_dict						)
								self.assertTrue(hasattr(lowerCamelCase,						'''do_resize'''						)						)
								self.assertTrue(hasattr(lowerCamelCase,						'''size'''						)						)
								self.assertTrue(hasattr(lowerCamelCase,						'''do_center_crop'''						)						)
								self.assertTrue(hasattr(lowerCamelCase,						'''crop_size'''						)						)




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								lowercase__     				=    self.image_processing_class.from_dict(self.image_processor_dict						)
								self.assertEqual(image_processor.size,						{'''shortest_edge''': 20}						)
								self.assertEqual(image_processor.crop_size,						{'''height''': 18, '''width''': 18}						)

								lowercase__     				=    self.image_processing_class.from_dict(self.image_processor_dict,						size=42,						crop_size=84						)
								self.assertEqual(image_processor.size,						{'''shortest_edge''': 42}						)
								self.assertEqual(image_processor.crop_size,						{'''height''': 84, '''width''': 84}						)




				def        lowercase__     (  self		:					Optional[int]						):

								'''simple docstring'''
								pass




				def        lowercase__     (  self		:					Any						):

								'''simple docstring'''
								# Initialize image_processing
								lowercase__     				=    self.image_processing_class(**self.image_processor_dict						)
								# create random PIL images
								lowercase__     				=    prepare_image_inputs(self.image_processor_tester,						equal_resolution=lowerCamelCase						)
								for image in image_inputs:
												self.assertIsInstance(lowerCamelCase,						Image.Image						)

								# Test not batched input
								lowercase__     				=    image_processing(image_inputs[0],						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)

								# Test batched
								lowercase__     				=    image_processing(lowerCamelCase,						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)




				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								# Initialize image_processing
								lowercase__     				=    self.image_processing_class(**self.image_processor_dict						)
								# create random numpy tensors
								lowercase__     				=    prepare_image_inputs(self.image_processor_tester,						equal_resolution=lowerCamelCase,						numpify=lowerCamelCase						)
								for image in image_inputs:
												self.assertIsInstance(lowerCamelCase,						np.ndarray						)

								# Test not batched input
								lowercase__     				=    image_processing(image_inputs[0],						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)

								# Test batched
								lowercase__     				=    image_processing(lowerCamelCase,						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)




				def        lowercase__     (  self		:					str						):

								'''simple docstring'''
								# Initialize image_processing
								lowercase__     				=    self.image_processing_class(**self.image_processor_dict						)
								# create random PyTorch tensors
								lowercase__     				=    prepare_image_inputs(self.image_processor_tester,						equal_resolution=lowerCamelCase,						torchify=lowerCamelCase						)
								for image in image_inputs:
												self.assertIsInstance(lowerCamelCase,						torch.Tensor						)

								# Test not batched input
								lowercase__     				=    image_processing(image_inputs[0],						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        1,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)

								# Test batched
								lowercase__     				=    image_processing(lowerCamelCase,						return_tensors='''pt'''						).pixel_values
								self.assertEqual(
								    encoded_images.shape,						(
								        self.image_processor_tester.batch_size,
								        self.image_processor_tester.num_channels,
								        self.image_processor_tester.crop_size['''height'''],
								        self.image_processor_tester.crop_size['''width'''],
								    ),						)




