


'''simple docstring'''




import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig

def 			__UpperCamelCase    (    _A					,   _A					,   _A=0				)    ->      Optional[int]:




	"""simple docstring"""





	if name is None:
		lowerCAmelCase					:    Optional[int]      							= None
	else:
		lowerCAmelCase					:    Any      							= '.' * max(0					,   spaces - 2				) + '# {:' + str(50 - spaces				) + 's}'
		lowerCAmelCase					:    Dict      							= fmt.format(_A				)

	# Print and recurse (if needed).
	if isinstance(_A					,   _A				):
		if msg is not None:
			print(_A				)
		for k in val.keys():
			recursive_print(_A					,   val[k]					,   spaces + 2				)
	elif isinstance(_A					,   torch.Tensor				):
		print(_A					,   ':'					,   val.size()				)
	else:
		print(_A					,   ':'					,   _A				)

def 			__UpperCamelCase    (    _A					,   _A					,   _A					,   _A					,   _A				)    ->      Any:




	"""simple docstring"""





	lowerCAmelCase					:    Optional[Any]      							= param.size()
	if checkpoint_version == 1.0:
		# version 1.0 stores [num_heads * hidden_size * num_splits, :]
		lowerCAmelCase					:    Tuple      							= (num_heads, hidden_size, num_splits) + input_shape[1:]
		lowerCAmelCase					:    Tuple      							= param.view(*_A				)
		lowerCAmelCase					:    Any      							= param.transpose(0					,   2				)
		lowerCAmelCase					:    Union[str, Any]      							= param.transpose(1					,   2				).contiguous()
	elif checkpoint_version >= 2.0:
		# other versions store [num_heads * num_splits * hidden_size, :]
		lowerCAmelCase					:    Tuple      							= (num_heads, num_splits, hidden_size) + input_shape[1:]
		lowerCAmelCase					:    Any      							= param.view(*_A				)
		lowerCAmelCase					:    str      							= param.transpose(0					,   1				).contiguous()
	lowerCAmelCase					:    Optional[int]      							= param.view(*_A				)
	return param

def 			__UpperCamelCase    (    _A					,   _A					,   _A				)    ->      str:




	"""simple docstring"""





	lowerCAmelCase					:    Optional[Any]      							= {}

	# old versions did not store training args
	lowerCAmelCase					:    Any      							= input_state_dict.get('args'					,   _A				)
	if ds_args is not None:
		# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
		# from pprint import pprint
		# pprint(vars(ds_args))

		lowerCAmelCase					:    Optional[int]      							= ds_args.padded_vocab_size
		lowerCAmelCase					:    Tuple      							= ds_args.max_position_embeddings
		lowerCAmelCase					:    Dict      							= ds_args.hidden_size
		lowerCAmelCase					:    Optional[int]      							= ds_args.num_layers
		lowerCAmelCase					:    Union[str, Any]      							= ds_args.num_attention_heads
		lowerCAmelCase					:    str      							= ds_args.ffn_hidden_size
		# pprint(config)

	# The number of heads.
	lowerCAmelCase					:    str      							= config.n_head
	# The hidden_size per head.
	lowerCAmelCase					:    Optional[Any]      							= config.n_embd // config.n_head
	# Megatron-LM checkpoint version
	if "checkpoint_version" in input_state_dict.keys():
		lowerCAmelCase					:    int      							= input_state_dict['checkpoint_version']
	else:
		lowerCAmelCase					:    Optional[Any]      							= 0.0

	# The model.
	lowerCAmelCase					:    Tuple      							= input_state_dict['model']
	# The language model.
	lowerCAmelCase					:    int      							= model['language_model']
	# The embeddings.
	lowerCAmelCase					:    Dict      							= lm['embedding']

	# The word embeddings.
	lowerCAmelCase					:    Union[str, Any]      							= embeddings['word_embeddings']['weight']
	# Truncate the embedding table to vocab_size rows.
	lowerCAmelCase					:    List[str]      							= word_embeddings[: config.vocab_size, :]
	lowerCAmelCase					:    int      							= word_embeddings

	# The position embeddings.
	lowerCAmelCase					:    Optional[Any]      							= embeddings['position_embeddings']['weight']
	# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
	lowerCAmelCase					:    int      							= pos_embeddings.size(0				)
	if n_positions != config.n_positions:
		raise ValueError(
		    F"pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"				)
	# Store the position embeddings.
	lowerCAmelCase					:    int      							= pos_embeddings

	# The transformer.
	lowerCAmelCase					:    Union[str, Any]      							= lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

	# The regex to extract layer names.
	lowerCAmelCase					:    Dict      							= re.compile(r'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'				)

	# The simple map of names for "automated" rules.
	lowerCAmelCase					:    str      							= {
	    'attention.dense': '.attn.c_proj.',
	    'self_attention.dense': '.attn.c_proj.',
	    'mlp.dense_h_to_4h': '.mlp.c_fc.',
	    'mlp.dense_4h_to_h': '.mlp.c_proj.',
	}

	# Extract the layers.
	for key, val in transformer.items():
		# Match the name.
		lowerCAmelCase					:    Optional[Any]      							= layer_re.match(_A				)

		# Stop if that's not a layer
		if m is None:
			break

		# The index of the layer.
		lowerCAmelCase					:    int      							= int(m.group(1				)				)
		# The name of the operation.
		lowerCAmelCase					:    str      							= m.group(2				)
		# Is it a weight or a bias?
		lowerCAmelCase					:    Tuple      							= m.group(3				)

		# The name of the layer.
		lowerCAmelCase					:    Any      							= F"transformer.h.{layer_idx}"

		# For layernorm(s), simply store the layer norm.
		if op_name.endswith('layernorm'				):
			lowerCAmelCase					:    Tuple      							= 'ln_1' if op_name.startswith('input'				) else 'ln_2'
			lowerCAmelCase					:    str      							= val

		# Transpose the QKV matrix.
		elif (
		    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
		) and weight_or_bias == "weight":
			# Insert a tensor of 1x1xDxD bias.
			lowerCAmelCase					:    int      							= torch.tril(torch.ones((n_positions, n_positions)					,   dtype=torch.floataa				)				).view(
			    1					,   1					,   _A					,   _A				)
			lowerCAmelCase					:    Optional[int]      							= causal_mask

			# Insert a "dummy" tensor for masked_bias.
			lowerCAmelCase					:    Optional[int]      							= torch.tensor(-1e4					,   dtype=torch.floataa				)
			lowerCAmelCase					:    str      							= masked_bias

			lowerCAmelCase					:    int      							= fix_query_key_value_ordering(_A					,   _A					,   3					,   _A					,   _A				)
			# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
			lowerCAmelCase					:    Optional[Any]      							= out_val.transpose(0					,   1				).contiguous()
			# Store.
			lowerCAmelCase					:    List[Any]      							= out_val

		# Transpose the bias.
		elif (
		    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
		) and weight_or_bias == "bias":
			lowerCAmelCase					:    Optional[Any]      							= fix_query_key_value_ordering(_A					,   _A					,   3					,   _A					,   _A				)
			# Store. No change of shape.
			lowerCAmelCase					:    List[Any]      							= out_val

		# Transpose the weights.
		elif weight_or_bias == "weight":
			lowerCAmelCase					:    List[str]      							= megatron_to_transformers[op_name]
			lowerCAmelCase					:    Optional[Any]      							= val.transpose(0					,   1				)

		# Copy the bias.
		elif weight_or_bias == "bias":
			lowerCAmelCase					:    Union[str, Any]      							= megatron_to_transformers[op_name]
			lowerCAmelCase					:    Tuple      							= val

    # DEBUG.
	assert config.n_layer == layer_idx + 1

	# The final layernorm.
	lowerCAmelCase					:    Tuple      							= transformer['final_layernorm.weight']
	lowerCAmelCase					:    List[Any]      							= transformer['final_layernorm.bias']

	# For LM head, transformers' wants the matrix to weight embeddings.
	lowerCAmelCase					:    List[str]      							= word_embeddings

	# It should be done!
	return output_state_dict

def 			__UpperCamelCase    (    )    ->      str:




	"""simple docstring"""





	lowerCAmelCase					:    int      							= argparse.ArgumentParser()
	parser.add_argument('--print-checkpoint-structure'					,   action='store_true'				)
	parser.add_argument(
	    'path_to_checkpoint'					,   type=_A					,   help='Path to the checkpoint file (.zip archive or direct .pt file)'					,   )
	parser.add_argument(
	    '--config_file'					,   default=''					,   type=_A					,   help='An optional config json file describing the pre-trained model.'					,   )
	lowerCAmelCase					:    List[str]      							= parser.parse_args()

	# Extract the basename.
	lowerCAmelCase					:    Optional[Any]      							= os.path.dirname(args.path_to_checkpoint				)

	# Load the model.
	# the .zip is very optional, let's keep it for backward compatibility
	print(F"Extracting PyTorch state dictionary from {args.path_to_checkpoint}"				)
	if args.path_to_checkpoint.endswith('.zip'				):
		with zipfile.ZipFile(args.path_to_checkpoint					,   'r'				) as checkpoint:
			with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'				) as pytorch_dict:
				lowerCAmelCase					:    List[Any]      							= torch.load(_A					,   map_location='cpu'				)
	else:
		lowerCAmelCase					:    Any      							= torch.load(args.path_to_checkpoint					,   map_location='cpu'				)

	lowerCAmelCase					:    str      							= input_state_dict.get('args'					,   _A				)

	# Read the config, or default to the model released by NVIDIA.
	if args.config_file == "":
		if ds_args is not None:
			if ds_args.bias_gelu_fusion:
				lowerCAmelCase					:    int      							= 'gelu_fast'
			elif ds_args.openai_gelu:
				lowerCAmelCase					:    List[str]      							= 'gelu_new'
			else:
				lowerCAmelCase					:    Any      							= 'gelu'
		else:
			# in the very early days this used to be "gelu_new"
			lowerCAmelCase					:    Union[str, Any]      							= 'gelu_new'

		# Spell out all parameters in case the defaults change.
		lowerCAmelCase					:    List[Any]      							= GPTaConfig(
		    vocab_size=5_02_57					,   n_positions=10_24					,   n_embd=10_24					,   n_layer=24					,   n_head=16					,   n_inner=40_96					,   activation_function=_A					,   resid_pdrop=0.1					,   embd_pdrop=0.1					,   attn_pdrop=0.1					,   layer_norm_epsilon=1e-5					,   initializer_range=0.02					,   summary_type='cls_index'					,   summary_use_proj=_A					,   summary_activation=_A					,   summary_proj_to_labels=_A					,   summary_first_dropout=0.1					,   scale_attn_weights=_A					,   use_cache=_A					,   bos_token_id=5_02_56					,   eos_token_id=5_02_56					,   )
	else:
		lowerCAmelCase					:    Dict      							= GPTaConfig.from_json_file(args.config_file				)

	lowerCAmelCase					:    str      							= ['GPT2LMHeadModel']

	# Convert.
	print('Converting'				)
	lowerCAmelCase					:    List[Any]      							= convert_megatron_checkpoint(_A					,   _A					,   _A				)

	# Print the structure of converted state dict.
	if args.print_checkpoint_structure:
		recursive_print(_A					,   _A				)

	# Add tokenizer class info to config
	# see https://github.com/huggingface/transformers/issues/13906)
	if ds_args is not None:
		lowerCAmelCase					:    Tuple      							= ds_args.tokenizer_type
		if tokenizer_type == "GPT2BPETokenizer":
			lowerCAmelCase					:    Dict      							= 'gpt2'
		elif tokenizer_type == "PretrainedFromHF":
			lowerCAmelCase					:    int      							= ds_args.tokenizer_name_or_path
		else:
			raise ValueError(F"Unrecognized tokenizer_type {tokenizer_type}"				)
	else:
		lowerCAmelCase					:    List[str]      							= 'gpt2'

	lowerCAmelCase					:    List[str]      							= AutoTokenizer.from_pretrained(_A				)
	lowerCAmelCase					:    Dict      							= type(_A				).__name__
	lowerCAmelCase					:    str      							= tokenizer_class

	# Store the config to file.
	print('Saving config'				)
	config.save_pretrained(_A				)

	# Save tokenizer based on args
	print(F"Adding {tokenizer_class} tokenizer files"				)
	tokenizer.save_pretrained(_A				)

	# Store the state_dict to file.
	lowerCAmelCase					:    Tuple      							= os.path.join(_A					,   'pytorch_model.bin'				)
	print(F"Saving checkpoint to \"{output_checkpoint_file}\""				)
	torch.save(_A					,   _A				)


####################################################################################################

if __name__ == "__main__":
				main()

####################################################################################################




'''simple docstring'''




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowerCAmelCase  :		Any		     =	logging.get_logger(__name__)

_lowerCAmelCase  :		List[str]		     =	{
    'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/config.json',
    'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/config.json',
    'xlm-roberta-large-finetuned-conll02-dutch': (
        'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/config.json'
    ),
    'xlm-roberta-large-finetuned-conll02-spanish': (
        'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/config.json'
    ),
    'xlm-roberta-large-finetuned-conll03-english': (
        'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/config.json'
    ),
    'xlm-roberta-large-finetuned-conll03-german': (
        'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/config.json'
    ),
}



class  lowerCAmelCase				(				a							):
	_lowerCamelCase   :					List[str]			      = """xlm-roberta"""



	def __init__(					self ,					snake_case__=3_0522 ,					snake_case__=768 ,					snake_case__=12 ,					snake_case__=12 ,					snake_case__=3072 ,					snake_case__="gelu" ,					snake_case__=0.1 ,					snake_case__=0.1 ,					snake_case__=512 ,					snake_case__=2 ,					snake_case__=0.0_2 ,					snake_case__=1e-1_2 ,					snake_case__=1 ,					snake_case__=0 ,					snake_case__=2 ,					snake_case__="absolute" ,					snake_case__=True ,					snake_case__=None ,					**snake_case__ ,					):
		super().__init__(pad_token_id=snake_case__ ,					bos_token_id=snake_case__ ,					eos_token_id=snake_case__ ,					**snake_case__    )

		lowerCAmelCase					:    Optional[Any]      							= vocab_size
		lowerCAmelCase					:    Optional[Any]      							= hidden_size
		lowerCAmelCase					:    Optional[Any]      							= num_hidden_layers
		lowerCAmelCase					:    Any      							= num_attention_heads
		lowerCAmelCase					:    Optional[int]      							= hidden_act
		lowerCAmelCase					:    Optional[int]      							= intermediate_size
		lowerCAmelCase					:    Dict      							= hidden_dropout_prob
		lowerCAmelCase					:    Union[str, Any]      							= attention_probs_dropout_prob
		lowerCAmelCase					:    Optional[Any]      							= max_position_embeddings
		lowerCAmelCase					:    Optional[int]      							= type_vocab_size
		lowerCAmelCase					:    int      							= initializer_range
		lowerCAmelCase					:    List[Any]      							= layer_norm_eps
		lowerCAmelCase					:    Union[str, Any]      							= position_embedding_type
		lowerCAmelCase					:    Union[str, Any]      							= use_cache
		lowerCAmelCase					:    List[str]      							= classifier_dropout



class  lowerCAmelCase				(				a							):



	@property
	def 				lowercase     (					self    ):
		if self.task == "multiple-choice":
			lowerCAmelCase					:    str      							= {0: 'batch', 1: 'choice', 2: 'sequence'}
		else:
			lowerCAmelCase					:    Optional[int]      							= {0: 'batch', 1: 'sequence'}
		return OrderedDict(
		    [
		        ('input_ids', dynamic_axis),
		        ('attention_mask', dynamic_axis),
		    ]    )
