"""simple docstring"""




from typing import List, Optional, Tuple, Union

import torch
from torch import nn
from torch.nn import CrossEntropyLoss

from ... import AutoBackbone
from ...modeling_outputs import SemanticSegmenterOutput
from ...modeling_utils import PreTrainedModel
from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings
from ...utils.backbone_utils import BackboneMixin
from .configuration_upernet import UperNetConfig


__lowerCAmelCase    :							Optional[Any]													=						[
    "openmmlab/upernet-convnext-tiny",
    # See all UperNet models at https://huggingface.co/models?filter=upernet
]

# General docstring
__lowerCAmelCase    :							int													=						"UperNetConfig"

class       a_      (     nn.Module			):




					def __init__(	self						:     Dict  ,						snake_case__						:     int  ,						snake_case__						:     int  ,						snake_case__						:     Union[int, Tuple[int, int]]  ,						snake_case__						:     Union[int, Tuple[int, int], str] = 0  ,						snake_case__						:     bool = False  ,						snake_case__						:     Union[int, Tuple[int, int]] = 1  ,						):
							super().__init__()
							lowerCAmelCase__        =   nn.Convad(
							    in_channels=snake_case__  ,						out_channels=snake_case__  ,						kernel_size=snake_case__  ,						padding=snake_case__  ,						bias=snake_case__  ,						dilation=snake_case__  ,						)
							lowerCAmelCase__        =   nn.BatchNormad(snake_case__     )
							lowerCAmelCase__        =   nn.ReLU()




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]  ,						snake_case__						:     torch.Tensor     ):
							lowerCAmelCase__        =   self.conv(snake_case__     )
							lowerCAmelCase__        =   self.batch_norm(snake_case__     )
							lowerCAmelCase__        =   self.activation(snake_case__     )

							return output

class       a_      (     nn.Module			):




					def __init__(	self						:     Any  ,						snake_case__						:     int  ,						snake_case__						:     int  ,						snake_case__						:     int     ):
							super().__init__()
							lowerCAmelCase__        =   [
							    nn.AdaptiveAvgPoolad(snake_case__     ),
							    UperNetConvModule(snake_case__  ,						snake_case__  ,						kernel_size=1     ),
							]
							for i, layer in enumerate(self.layers     ):
									self.add_module(str(snake_case__     )  ,						snake_case__     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Tuple  ,						snake_case__						:     torch.Tensor     ):
							lowerCAmelCase__        =   input
							for layer in self.layers:
									lowerCAmelCase__        =   layer(snake_case__     )
							return hidden_state

class       a_      (     nn.Module			):




					def __init__(	self						:     Optional[int]  ,						snake_case__						:     Tuple[int, ...]  ,						snake_case__						:     int  ,						snake_case__						:     int  ,						snake_case__						:     bool     ):
							super().__init__()
							lowerCAmelCase__        =   pool_scales
							lowerCAmelCase__        =   align_corners
							lowerCAmelCase__        =   in_channels
							lowerCAmelCase__        =   channels
							lowerCAmelCase__        =   []
							for i, pool_scale in enumerate(snake_case__     ):
									lowerCAmelCase__        =   UperNetPyramidPoolingBlock(pool_scale=snake_case__  ,						in_channels=snake_case__  ,						channels=snake_case__     )
									self.blocks.append(snake_case__     )
									self.add_module(str(snake_case__     )  ,						snake_case__     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]  ,						snake_case__						:     torch.Tensor     ):
							lowerCAmelCase__        =   []
							for ppm in self.blocks:
									lowerCAmelCase__        =   ppm(snake_case__     )
									lowerCAmelCase__        =   nn.functional.interpolate(
									    snake_case__  ,						size=x.size()[2:]  ,						mode="""bilinear"""  ,						align_corners=self.align_corners     )
									ppm_outs.append(snake_case__     )
							return ppm_outs

class       a_      (     nn.Module			):




					def __init__(	self						:     Union[str, Any]  ,						snake_case__						:     int  ,						snake_case__						:     Optional[Any]     ):
							super().__init__()

							lowerCAmelCase__        =   config
							lowerCAmelCase__        =   config.pool_scales  # e.g. (1, 2, 3, 6)
							lowerCAmelCase__        =   in_channels
							lowerCAmelCase__        =   config.hidden_size
							lowerCAmelCase__        =   False
							lowerCAmelCase__        =   nn.Convad(self.channels  ,						config.num_labels  ,						kernel_size=1     )

							# PSP Module
							lowerCAmelCase__        =   UperNetPyramidPoolingModule(
							    self.pool_scales  ,						self.in_channels[-1]  ,						self.channels  ,						align_corners=self.align_corners  ,						)
							lowerCAmelCase__        =   UperNetConvModule(
							    self.in_channels[-1] + len(self.pool_scales     ) * self.channels  ,						self.channels  ,						kernel_size=3  ,						padding=1  ,						)
							# FPN Module
							lowerCAmelCase__        =   nn.ModuleList()
							lowerCAmelCase__        =   nn.ModuleList()
							for in_channels in self.in_channels[:-1]:  # skip the top layer
									lowerCAmelCase__        =   UperNetConvModule(snake_case__  ,						self.channels  ,						kernel_size=1     )
									lowerCAmelCase__        =   UperNetConvModule(self.channels  ,						self.channels  ,						kernel_size=3  ,						padding=1     )
									self.lateral_convs.append(snake_case__     )
									self.fpn_convs.append(snake_case__     )

							lowerCAmelCase__        =   UperNetConvModule(
							    len(self.in_channels     ) * self.channels  ,						self.channels  ,						kernel_size=3  ,						padding=1  ,						)




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Optional[int]     ):
							self.apply(self._init_weights     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     List[str]  ,						snake_case__						:     List[Any]     ):
							if isinstance(snake_case__  ,						nn.Convad     ):
									module.weight.data.normal_(mean=0.0  ,						std=self.config.initializer_range     )
									if module.bias is not None:
											module.bias.data.zero_()




					def 						_SCREAMING_SNAKE_CASE						(	self						:     List[Any]  ,						snake_case__						:     Optional[int]     ):
							lowerCAmelCase__        =   inputs[-1]
							lowerCAmelCase__        =   [x]
							psp_outs.extend(self.psp_modules(snake_case__     )     )
							lowerCAmelCase__        =   torch.cat(snake_case__  ,						dim=1     )
							lowerCAmelCase__        =   self.bottleneck(snake_case__     )

							return output




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Tuple  ,						snake_case__						:     torch.Tensor     ):
							# build laterals
							lowerCAmelCase__        =   [lateral_conv(encoder_hidden_states[i]     ) for i, lateral_conv in enumerate(self.lateral_convs     )]

							laterals.append(self.psp_forward(snake_case__     )     )

							# build top-down path
							lowerCAmelCase__        =   len(snake_case__     )
							for i in range(used_backbone_levels - 1  ,						0  ,						-1     ):
									lowerCAmelCase__        =   laterals[i - 1].shape[2:]
									lowerCAmelCase__        =   laterals[i - 1] + nn.functional.interpolate(
									    laterals[i]  ,						size=snake_case__  ,						mode="""bilinear"""  ,						align_corners=self.align_corners     )

							# build outputs
							lowerCAmelCase__        =   [self.fpn_convs[i](laterals[i]     ) for i in range(used_backbone_levels - 1     )]
							# append psp feature
							fpn_outs.append(laterals[-1]     )

							for i in range(used_backbone_levels - 1  ,						0  ,						-1     ):
									lowerCAmelCase__        =   nn.functional.interpolate(
									    fpn_outs[i]  ,						size=fpn_outs[0].shape[2:]  ,						mode="""bilinear"""  ,						align_corners=self.align_corners     )
							lowerCAmelCase__        =   torch.cat(snake_case__  ,						dim=1     )
							lowerCAmelCase__        =   self.fpn_bottleneck(snake_case__     )
							lowerCAmelCase__        =   self.classifier(snake_case__     )

							return output

class       a_      (     nn.Module			):




					def __init__(	self						:     Optional[int]  ,						snake_case__						:     List[str]  ,						snake_case__						:     int = 2  ,						snake_case__						:     int = 3  ,						snake_case__						:     Union[int, Tuple[int, int]] = 1     ):
							super().__init__()

							lowerCAmelCase__        =   config
							lowerCAmelCase__        =   config.auxiliary_in_channels
							lowerCAmelCase__        =   config.auxiliary_channels
							lowerCAmelCase__        =   config.auxiliary_num_convs
							lowerCAmelCase__        =   config.auxiliary_concat_input
							lowerCAmelCase__        =   in_index

							lowerCAmelCase__        =   (kernel_size // 2) * dilation
							lowerCAmelCase__        =   []
							convs.append(
							    UperNetConvModule(
							        self.in_channels  ,						self.channels  ,						kernel_size=snake_case__  ,						padding=snake_case__  ,						dilation=snake_case__     )     )
							for i in range(self.num_convs - 1     ):
									convs.append(
									    UperNetConvModule(
									        self.channels  ,						self.channels  ,						kernel_size=snake_case__  ,						padding=snake_case__  ,						dilation=snake_case__     )     )
							if self.num_convs == 0:
									lowerCAmelCase__        =   nn.Identity()
							else:
									lowerCAmelCase__        =   nn.Sequential(*snake_case__     )
							if self.concat_input:
									lowerCAmelCase__        =   UperNetConvModule(
									    self.in_channels + self.channels  ,						self.channels  ,						kernel_size=snake_case__  ,						padding=kernel_size // 2     )

							lowerCAmelCase__        =   nn.Convad(self.channels  ,						config.num_labels  ,						kernel_size=1     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Dict     ):
							self.apply(self._init_weights     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Any  ,						snake_case__						:     Union[str, Any]     ):
							if isinstance(snake_case__  ,						nn.Convad     ):
									module.weight.data.normal_(mean=0.0  ,						std=self.config.initializer_range     )
									if module.bias is not None:
											module.bias.data.zero_()




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]  ,						snake_case__						:     torch.Tensor     ):
							# just take the relevant feature maps
							lowerCAmelCase__        =   encoder_hidden_states[self.in_index]
							lowerCAmelCase__        =   self.convs(snake_case__     )
							if self.concat_input:
									lowerCAmelCase__        =   self.conv_cat(torch.cat([hidden_states, output]  ,						dim=1     )     )
							lowerCAmelCase__        =   self.classifier(snake_case__     )
							return output

class       a_      (     __UpperCamelCase			):
					UpperCamelCase_ :	Dict			=    UperNetConfig
					UpperCamelCase_ :	Tuple			=    "pixel_values"
					UpperCamelCase_ :	str			=    True




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]  ,						snake_case__						:     Tuple     ):
							if isinstance(snake_case__  ,						snake_case__     ):
									module.backbone.init_weights()
									module.decode_head.init_weights()
									module.auxiliary_head.init_weights()




					def 						_SCREAMING_SNAKE_CASE						(	self						:     str     ):
							self.backbone.init_weights()
							self.decode_head.init_weights()
							self.auxiliary_head.init_weights()




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Optional[Any]  ,						snake_case__						:     Tuple  ,						snake_case__						:     Any=False     ):
							if isinstance(snake_case__  ,						snake_case__     ):
									lowerCAmelCase__        =   value
__lowerCAmelCase    :							Optional[int]													=						R"\n    Parameters:\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use\n    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n        config ([`UperNetConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n"

__lowerCAmelCase    :							List[Any]													=						R"\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using\n            [`AutoImageProcessor`]. See [`SegformerImageProcessor.__call__`] for details.\n        output_attentions (`bool`, *optional*):\n            Whether or not to return the attentions tensors of all attention layers in case the backbone has them. See\n            `attentions` under returned tensors for more detail.\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers of the backbone. See `hidden_states` under\n            returned tensors for more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n"

@add_start_docstrings(
    "UperNet framework leveraging any vision backbone e.g. for ADE20k, CityScapes." ,		__UpperCamelCase ,		)
class       a_      (     __UpperCamelCase			):




					def __init__(	self						:     Dict  ,						snake_case__						:     Optional[Any]     ):
							super().__init__(snake_case__     )

							lowerCAmelCase__        =   AutoBackbone.from_config(config.backbone_config     )

							# Semantic segmentation head(s)
							lowerCAmelCase__        =   UperNetHead(snake_case__  ,						in_channels=self.backbone.channels     )
							lowerCAmelCase__        =   UperNetFCNHead(snake_case__     ) if config.use_auxiliary_head else None

							# Initialize weights and apply final processing
							self.post_init()




					@add_start_docstrings_to_model_forward(UPERNET_INPUTS_DOCSTRING.format("""batch_size, sequence_length"""     )     )
					@replace_return_docstrings(output_type=snake_case__  ,						config_class=_CONFIG_FOR_DOC     )
					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]  ,						snake_case__						:     Optional[torch.Tensor] = None  ,						snake_case__						:     Optional[bool] = None  ,						snake_case__						:     Optional[bool] = None  ,						snake_case__						:     Optional[torch.Tensor] = None  ,						snake_case__						:     Optional[bool] = None  ,						):
							lowerCAmelCase__        =   return_dict if return_dict is not None else self.config.use_return_dict
							lowerCAmelCase__        =   (
							    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
							)
							lowerCAmelCase__        =   output_attentions if output_attentions is not None else self.config.output_attentions

							lowerCAmelCase__        =   self.backbone.forward_with_filtered_kwargs(
							    snake_case__  ,						output_hidden_states=snake_case__  ,						output_attentions=snake_case__     )
							lowerCAmelCase__        =   outputs.feature_maps

							lowerCAmelCase__        =   self.decode_head(snake_case__     )
							lowerCAmelCase__        =   nn.functional.interpolate(snake_case__  ,						size=pixel_values.shape[2:]  ,						mode="""bilinear"""  ,						align_corners=snake_case__     )

							lowerCAmelCase__        =   None
							if self.auxiliary_head is not None:
									lowerCAmelCase__        =   self.auxiliary_head(snake_case__     )
									lowerCAmelCase__        =   nn.functional.interpolate(
									    snake_case__  ,						size=pixel_values.shape[2:]  ,						mode="""bilinear"""  ,						align_corners=snake_case__     )

							lowerCAmelCase__        =   None
							if labels is not None:
									if self.config.num_labels == 1:
											raise ValueError("""The number of labels should be greater than one"""     )
									else:
											# compute weighted loss
											lowerCAmelCase__        =   CrossEntropyLoss(ignore_index=self.config.loss_ignore_index     )
											lowerCAmelCase__        =   loss_fct(snake_case__  ,						snake_case__     )
											lowerCAmelCase__        =   loss_fct(snake_case__  ,						snake_case__     )
											lowerCAmelCase__        =   main_loss + self.config.auxiliary_loss_weight * auxiliary_loss

							if not return_dict:
									if output_hidden_states:
											lowerCAmelCase__        =   (logits,) + outputs[1:]
									else:
											lowerCAmelCase__        =   (logits,) + outputs[2:]
									return ((loss,) + output) if loss is not None else output

							return SemanticSegmenterOutput(
							    loss=snake_case__  ,						logits=snake_case__  ,						hidden_states=outputs.hidden_states  ,						attentions=outputs.attentions  ,						)





"""simple docstring"""




import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse("0.8.3"):
			raise Exception("requires gluonnlp == 0.8.3")

if version.parse(mx.__version__) != version.parse("1.5.0"):
			raise Exception("requires mxnet == 1.5.0")

logging.set_verbosity_info()
__lowerCAmelCase    :							Any													=						logging.get_logger(__name__)

__lowerCAmelCase    :							Any													=						"The Nymphenburg Palace is a beautiful palace in Munich!"







def 							_UpperCAmelCase		(       lowerCamelCase__			,  lowerCamelCase__     ):





		"""simple docstring"""




		lowerCAmelCase__        =   {
		    """attention_cell""": """multi_head""",
		    """num_layers""": 4,
		    """units""": 1024,
		    """hidden_size""": 768,
		    """max_length""": 512,
		    """num_heads""": 8,
		    """scaled""": True,
		    """dropout""": 0.1,
		    """use_residual""": True,
		    """embed_size""": 1024,
		    """embed_dropout""": 0.1,
		    """word_embed""": None,
		    """layer_norm_eps""": 1e-5,
		    """token_type_vocab_size""": 2,
		}

		lowerCAmelCase__        =   bort_4_8_768_1024_hparams

		# Let's construct the original Bort model here
		# Taken from official BERT implementation, see:
		# https://github.com/alexa/bort/blob/master/bort/bort.py
		lowerCAmelCase__        =   BERTEncoder(
		    attention_cell=predefined_args["""attention_cell"""]			,  num_layers=predefined_args["""num_layers"""]			,  units=predefined_args["""units"""]			,  hidden_size=predefined_args["""hidden_size"""]			,  max_length=predefined_args["""max_length"""]			,  num_heads=predefined_args["""num_heads"""]			,  scaled=predefined_args["""scaled"""]			,  dropout=predefined_args["""dropout"""]			,  output_attention=lowerCamelCase__			,  output_all_encodings=lowerCamelCase__			,  use_residual=predefined_args["""use_residual"""]			,  activation=predefined_args.get("""activation"""			,  """gelu"""     )			,  layer_norm_eps=predefined_args.get("""layer_norm_eps"""			,  lowerCamelCase__     )			,  )

		# Vocab information needs to be fetched first
		# It's the same as RoBERTa, so RobertaTokenizer can be used later
		lowerCAmelCase__        =   """openwebtext_ccnews_stories_books_cased"""

		# Specify download folder to Gluonnlp's vocab
		lowerCAmelCase__        =   os.path.join(get_home_dir()			,  """models"""     )
		lowerCAmelCase__        =   _load_vocab(lowerCamelCase__			,  lowerCamelCase__			,  lowerCamelCase__			,  cls=lowerCamelCase__     )

		lowerCAmelCase__        =   nlp.model.BERTModel(
		    lowerCamelCase__			,  len(lowerCamelCase__     )			,  units=predefined_args["""units"""]			,  embed_size=predefined_args["""embed_size"""]			,  embed_dropout=predefined_args["""embed_dropout"""]			,  word_embed=predefined_args["""word_embed"""]			,  use_pooler=lowerCamelCase__			,  use_token_type_embed=lowerCamelCase__			,  token_type_vocab_size=predefined_args["""token_type_vocab_size"""]			,  use_classifier=lowerCamelCase__			,  use_decoder=lowerCamelCase__			,  )

		original_bort.load_parameters(lowerCamelCase__			,  cast_dtype=lowerCamelCase__			,  ignore_extra=lowerCamelCase__     )
		lowerCAmelCase__        =   original_bort._collect_params_with_prefix()

		# Build our config 🤗
		lowerCAmelCase__        =   {
		    """architectures""": ["""BertForMaskedLM"""],
		    """attention_probs_dropout_prob""": predefined_args["""dropout"""],
		    """hidden_act""": """gelu""",
		    """hidden_dropout_prob""": predefined_args["""dropout"""],
		    """hidden_size""": predefined_args["""embed_size"""],
		    """initializer_range""": 0.02,
		    """intermediate_size""": predefined_args["""hidden_size"""],
		    """layer_norm_eps""": predefined_args["""layer_norm_eps"""],
		    """max_position_embeddings""": predefined_args["""max_length"""],
		    """model_type""": """bort""",
		    """num_attention_heads""": predefined_args["""num_heads"""],
		    """num_hidden_layers""": predefined_args["""num_layers"""],
		    """pad_token_id""": 1,  # 2 = BERT, 1 = RoBERTa
		    """type_vocab_size""": 1,  # 2 = BERT, 1 = RoBERTa
		    """vocab_size""": len(lowerCamelCase__     ),
		}

		lowerCAmelCase__        =   BertConfig.from_dict(lowerCamelCase__     )
		lowerCAmelCase__        =   BertForMaskedLM(lowerCamelCase__     )
		hf_bort_model.eval()

		# Parameter mapping table (Gluonnlp to Transformers)
		# * denotes layer index
		#
		# | Gluon Parameter                                                | Transformers Parameter
		# | -------------------------------------------------------------- | ----------------------
		# | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
		# | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
		# | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
		# | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
		# | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
		# | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
		# | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
		# | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
		# | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
		# | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
		# | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
		# | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
		# | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
		# | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
		# | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
		# | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
		# | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
		# | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
		# | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
		# | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

		# Helper function to convert MXNET Arrays to PyTorch
		def to_torch(lowerCamelCase__     ) -> nn.Parameter:
				return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()     )     )

		# Check param shapes and map new HF param back
		def check_and_map_params(lowerCamelCase__			,  lowerCamelCase__     ):
				lowerCAmelCase__        =   hf_param.shape

				lowerCAmelCase__        =   to_torch(params[gluon_param]     )
				lowerCAmelCase__        =   gluon_param.shape

				assert (
				    shape_hf == shape_gluon
				), f"""The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers"""

				return gluon_param

		lowerCAmelCase__        =   check_and_map_params(
		    hf_bort_model.bert.embeddings.word_embeddings.weight			,  """word_embed.0.weight"""     )
		lowerCAmelCase__        =   check_and_map_params(
		    hf_bort_model.bert.embeddings.position_embeddings.weight			,  """encoder.position_weight"""     )
		lowerCAmelCase__        =   check_and_map_params(
		    hf_bort_model.bert.embeddings.LayerNorm.bias			,  """encoder.layer_norm.beta"""     )
		lowerCAmelCase__        =   check_and_map_params(
		    hf_bort_model.bert.embeddings.LayerNorm.weight			,  """encoder.layer_norm.gamma"""     )

		# Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
		lowerCAmelCase__        =   torch.zeros_like(
		    hf_bort_model.bert.embeddings.token_type_embeddings.weight.data     )

		for i in range(hf_bort_config.num_hidden_layers     ):
				lowerCAmelCase__        =   hf_bort_model.bert.encoder.layer[i]

				# self attention
				lowerCAmelCase__        =   layer.attention.self

				lowerCAmelCase__        =   check_and_map_params(
				    self_attn.key.bias.data			,  f"""encoder.transformer_cells.{i}.attention_cell.proj_key.bias"""     )

				lowerCAmelCase__        =   check_and_map_params(
				    self_attn.key.weight.data			,  f"""encoder.transformer_cells.{i}.attention_cell.proj_key.weight"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    self_attn.query.bias.data			,  f"""encoder.transformer_cells.{i}.attention_cell.proj_query.bias"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    self_attn.query.weight.data			,  f"""encoder.transformer_cells.{i}.attention_cell.proj_query.weight"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    self_attn.value.bias.data			,  f"""encoder.transformer_cells.{i}.attention_cell.proj_value.bias"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    self_attn.value.weight.data			,  f"""encoder.transformer_cells.{i}.attention_cell.proj_value.weight"""     )

				# self attention output
				lowerCAmelCase__        =   layer.attention.output

				lowerCAmelCase__        =   check_and_map_params(
				    self_output.dense.bias			,  f"""encoder.transformer_cells.{i}.proj.bias"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    self_output.dense.weight			,  f"""encoder.transformer_cells.{i}.proj.weight"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    self_output.LayerNorm.bias			,  f"""encoder.transformer_cells.{i}.layer_norm.beta"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    self_output.LayerNorm.weight			,  f"""encoder.transformer_cells.{i}.layer_norm.gamma"""     )

				# intermediate
				lowerCAmelCase__        =   layer.intermediate

				lowerCAmelCase__        =   check_and_map_params(
				    intermediate.dense.bias			,  f"""encoder.transformer_cells.{i}.ffn.ffn_1.bias"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    intermediate.dense.weight			,  f"""encoder.transformer_cells.{i}.ffn.ffn_1.weight"""     )

				# output
				lowerCAmelCase__        =   layer.output

				lowerCAmelCase__        =   check_and_map_params(
				    bert_output.dense.bias			,  f"""encoder.transformer_cells.{i}.ffn.ffn_2.bias"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    bert_output.dense.weight			,  f"""encoder.transformer_cells.{i}.ffn.ffn_2.weight"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    bert_output.LayerNorm.bias			,  f"""encoder.transformer_cells.{i}.ffn.layer_norm.beta"""     )
				lowerCAmelCase__        =   check_and_map_params(
				    bert_output.LayerNorm.weight			,  f"""encoder.transformer_cells.{i}.ffn.layer_norm.gamma"""     )

		# Save space and energy 🎄
		hf_bort_model.half()

		# Compare output of both models
		lowerCAmelCase__        =   RobertaTokenizer.from_pretrained("""roberta-base"""     )

		lowerCAmelCase__        =   tokenizer.encode_plus(lowerCamelCase__     )["""input_ids"""]

		# Get gluon output
		lowerCAmelCase__        =   mx.nd.array([input_ids]     )
		lowerCAmelCase__        =   original_bort(inputs=lowerCamelCase__			,  token_types=[]     )

		# Get Transformer output (save and reload model again)
		hf_bort_model.save_pretrained(lowerCamelCase__     )
		lowerCAmelCase__        =   BertModel.from_pretrained(lowerCamelCase__     )
		hf_bort_model.eval()

		lowerCAmelCase__        =   tokenizer.encode_plus(lowerCamelCase__			,  return_tensors="""pt"""     )
		lowerCAmelCase__        =   hf_bort_model(**lowerCamelCase__     )[0]

		lowerCAmelCase__        =   output_gluon[0].asnumpy()
		lowerCAmelCase__        =   output_hf[0].detach().numpy()

		lowerCAmelCase__        =   np.max(np.abs(hf_layer - gluon_layer     )     ).item()
		lowerCAmelCase__        =   np.allclose(lowerCamelCase__			,  lowerCamelCase__			,  atol=1e-3     )

		if success:
				print("""✔️ Both model do output the same tensors"""     )
		else:
				print("""❌ Both model do **NOT** output the same tensors"""     )
				print("""Absolute difference is:"""			,  lowerCamelCase__     )


if __name__ == "__main__":
			__lowerCAmelCase    :							Union[str, Any]													=						argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    "--bort_checkpoint_path", default=None, type=str, required=True, help="Path the official Bort params file."
			)
			parser.add_argument(
			    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
			)
			__lowerCAmelCase    :							str													=						parser.parse_args()
			convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)




