



# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig





def 							A			(  _UpperCAmelCase       :     Any							,     _UpperCAmelCase       :     Union[str, Any]							,     _UpperCAmelCase       :     Tuple=0					)       ->				Optional[Any]:







			'''simple docstring'''
			# Format the message.
			if name is None:
						_UpperCAmelCase	 =     None
			else:
						_UpperCAmelCase	 =     '.' * max(0							,     spaces - 2					) + '# {:' + str(50 - spaces					) + 's}'
						_UpperCAmelCase	 =     fmt.format(_UpperCAmelCase					)

			# Print and recurse (if needed).
			if isinstance(_UpperCAmelCase							,     _UpperCAmelCase					):
						if msg is not None:
									print(_UpperCAmelCase					)
						for k in val.keys():
									recursive_print(_UpperCAmelCase							,     val[k]							,     spaces + 2					)
			elif isinstance(_UpperCAmelCase							,     torch.Tensor					):
						print(_UpperCAmelCase							,     ':'							,     val.size()					)
			else:
						print(_UpperCAmelCase							,     ':'							,     _UpperCAmelCase					)





def 							A			(  _UpperCAmelCase       :     Union[str, Any]							,     _UpperCAmelCase       :     Dict							,     _UpperCAmelCase       :     Optional[int]							,     _UpperCAmelCase       :     Tuple							,     _UpperCAmelCase       :     Union[str, Any]					)       ->				Any:







			'''simple docstring'''
			_UpperCAmelCase	 =     param.size()
			if checkpoint_version == 1.0:
						# version 1.0 stores [num_heads * hidden_size * num_splits, :]
						_UpperCAmelCase	 =     (num_heads, hidden_size, num_splits) + input_shape[1:]
						_UpperCAmelCase	 =     param.view(*_UpperCAmelCase					)
						_UpperCAmelCase	 =     param.transpose(0							,     2					)
						_UpperCAmelCase	 =     param.transpose(1							,     2					).contiguous()
			elif checkpoint_version >= 2.0:
						# other versions store [num_heads * num_splits * hidden_size, :]
						_UpperCAmelCase	 =     (num_heads, num_splits, hidden_size) + input_shape[1:]
						_UpperCAmelCase	 =     param.view(*_UpperCAmelCase					)
						_UpperCAmelCase	 =     param.transpose(0							,     1					).contiguous()
			_UpperCAmelCase	 =     param.view(*_UpperCAmelCase					)
			return param





def 							A			(  _UpperCAmelCase       :     Optional[Any]							,     _UpperCAmelCase       :     List[str]							,     _UpperCAmelCase       :     Dict					)       ->				Optional[Any]:







			'''simple docstring'''
			_UpperCAmelCase	 =     {}

			# old versions did not store training args
			_UpperCAmelCase	 =     input_state_dict.get('args'							,     _UpperCAmelCase					)
			if ds_args is not None:
						# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
						# from pprint import pprint
						# pprint(vars(ds_args))

						_UpperCAmelCase	 =     ds_args.padded_vocab_size
						_UpperCAmelCase	 =     ds_args.max_position_embeddings
						_UpperCAmelCase	 =     ds_args.hidden_size
						_UpperCAmelCase	 =     ds_args.num_layers
						_UpperCAmelCase	 =     ds_args.num_attention_heads
						_UpperCAmelCase	 =     ds_args.ffn_hidden_size
						# pprint(config)

			# The number of heads.
			_UpperCAmelCase	 =     config.n_head
			# The hidden_size per head.
			_UpperCAmelCase	 =     config.n_embd // config.n_head
			# Megatron-LM checkpoint version
			if "checkpoint_version" in input_state_dict.keys():
						_UpperCAmelCase	 =     input_state_dict['checkpoint_version']
			else:
						_UpperCAmelCase	 =     0.0

			# The model.
			_UpperCAmelCase	 =     input_state_dict['model']
			# The language model.
			_UpperCAmelCase	 =     model['language_model']
			# The embeddings.
			_UpperCAmelCase	 =     lm['embedding']

			# The word embeddings.
			_UpperCAmelCase	 =     embeddings['word_embeddings']['weight']
			# Truncate the embedding table to vocab_size rows.
			_UpperCAmelCase	 =     word_embeddings[: config.vocab_size, :]
			_UpperCAmelCase	 =     word_embeddings

			# The position embeddings.
			_UpperCAmelCase	 =     embeddings['position_embeddings']['weight']
			# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
			_UpperCAmelCase	 =     pos_embeddings.size(0					)
			if n_positions != config.n_positions:
						raise ValueError(
						    F"pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"					)
			# Store the position embeddings.
			_UpperCAmelCase	 =     pos_embeddings

			# The transformer.
			_UpperCAmelCase	 =     lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

			# The regex to extract layer names.
			_UpperCAmelCase	 =     re.compile(R'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'					)

			# The simple map of names for "automated" rules.
			_UpperCAmelCase	 =     {
			    'attention.dense': '.attn.c_proj.',
			    'self_attention.dense': '.attn.c_proj.',
			    'mlp.dense_h_to_4h': '.mlp.c_fc.',
			    'mlp.dense_4h_to_h': '.mlp.c_proj.',
			}

			# Extract the layers.
			for key, val in transformer.items():
						# Match the name.
						_UpperCAmelCase	 =     layer_re.match(_UpperCAmelCase					)

						# Stop if that's not a layer
						if m is None:
									break

						# The index of the layer.
						_UpperCAmelCase	 =     int(m.group(1					)					)
						# The name of the operation.
						_UpperCAmelCase	 =     m.group(2					)
						# Is it a weight or a bias?
						_UpperCAmelCase	 =     m.group(3					)

						# The name of the layer.
						_UpperCAmelCase	 =     F"transformer.h.{layer_idx}"

						# For layernorm(s), simply store the layer norm.
						if op_name.endswith('layernorm'					):
									_UpperCAmelCase	 =     'ln_1' if op_name.startswith('input'					) else 'ln_2'
									_UpperCAmelCase	 =     val

						# Transpose the QKV matrix.
						elif (
						    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
						) and weight_or_bias == "weight":
									# Insert a tensor of 1x1xDxD bias.
									_UpperCAmelCase	 =     torch.tril(torch.ones((n_positions, n_positions)							,     dtype=torch.floataa					)					).view(
									    1							,     1							,     _UpperCAmelCase							,     _UpperCAmelCase					)
									_UpperCAmelCase	 =     causal_mask

									# Insert a "dummy" tensor for masked_bias.
									_UpperCAmelCase	 =     torch.tensor(-1E4							,     dtype=torch.floataa					)
									_UpperCAmelCase	 =     masked_bias

									_UpperCAmelCase	 =     fix_query_key_value_ordering(_UpperCAmelCase							,     _UpperCAmelCase							,     3							,     _UpperCAmelCase							,     _UpperCAmelCase					)
									# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
									_UpperCAmelCase	 =     out_val.transpose(0							,     1					).contiguous()
									# Store.
									_UpperCAmelCase	 =     out_val

						# Transpose the bias.
						elif (
						    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
						) and weight_or_bias == "bias":
									_UpperCAmelCase	 =     fix_query_key_value_ordering(_UpperCAmelCase							,     _UpperCAmelCase							,     3							,     _UpperCAmelCase							,     _UpperCAmelCase					)
									# Store. No change of shape.
									_UpperCAmelCase	 =     out_val

						# Transpose the weights.
						elif weight_or_bias == "weight":
									_UpperCAmelCase	 =     megatron_to_transformers[op_name]
									_UpperCAmelCase	 =     val.transpose(0							,     1					)

						# Copy the bias.
						elif weight_or_bias == "bias":
									_UpperCAmelCase	 =     megatron_to_transformers[op_name]
									_UpperCAmelCase	 =     val

    # DEBUG.
			assert config.n_layer == layer_idx + 1

			# The final layernorm.
			_UpperCAmelCase	 =     transformer['final_layernorm.weight']
			_UpperCAmelCase	 =     transformer['final_layernorm.bias']

			# For LM head, transformers' wants the matrix to weight embeddings.
			_UpperCAmelCase	 =     word_embeddings

			# It should be done!
			return output_state_dict





def 							A			(  )       ->				Dict:







			'''simple docstring'''
			_UpperCAmelCase	 =     argparse.ArgumentParser()
			parser.add_argument('--print-checkpoint-structure'							,     action='store_true'					)
			parser.add_argument(
			    'path_to_checkpoint'							,     type=_UpperCAmelCase							,     help='Path to the checkpoint file (.zip archive or direct .pt file)'							,     )
			parser.add_argument(
			    '--config_file'							,     default=''							,     type=_UpperCAmelCase							,     help='An optional config json file describing the pre-trained model.'							,     )
			_UpperCAmelCase	 =     parser.parse_args()

			# Extract the basename.
			_UpperCAmelCase	 =     os.path.dirname(args.path_to_checkpoint					)

			# Load the model.
			# the .zip is very optional, let's keep it for backward compatibility
			print(F"Extracting PyTorch state dictionary from {args.path_to_checkpoint}"					)
			if args.path_to_checkpoint.endswith('.zip'					):
						with zipfile.ZipFile(args.path_to_checkpoint							,     'r'					) as checkpoint:
									with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'					) as pytorch_dict:
												_UpperCAmelCase	 =     torch.load(_UpperCAmelCase							,     map_location='cpu'					)
			else:
						_UpperCAmelCase	 =     torch.load(args.path_to_checkpoint							,     map_location='cpu'					)

			_UpperCAmelCase	 =     input_state_dict.get('args'							,     _UpperCAmelCase					)

			# Read the config, or default to the model released by NVIDIA.
			if args.config_file == "":
						if ds_args is not None:
									if ds_args.bias_gelu_fusion:
												_UpperCAmelCase	 =     'gelu_fast'
									elif ds_args.openai_gelu:
												_UpperCAmelCase	 =     'gelu_new'
									else:
												_UpperCAmelCase	 =     'gelu'
						else:
									# in the very early days this used to be "gelu_new"
									_UpperCAmelCase	 =     'gelu_new'

						# Spell out all parameters in case the defaults change.
						_UpperCAmelCase	 =     GPTaConfig(
						    vocab_size=50_257							,     n_positions=1_024							,     n_embd=1_024							,     n_layer=24							,     n_head=16							,     n_inner=4_096							,     activation_function=_UpperCAmelCase							,     resid_pdrop=0.1							,     embd_pdrop=0.1							,     attn_pdrop=0.1							,     layer_norm_epsilon=1E-5							,     initializer_range=0.02							,     summary_type='cls_index'							,     summary_use_proj=_UpperCAmelCase							,     summary_activation=_UpperCAmelCase							,     summary_proj_to_labels=_UpperCAmelCase							,     summary_first_dropout=0.1							,     scale_attn_weights=_UpperCAmelCase							,     use_cache=_UpperCAmelCase							,     bos_token_id=50_256							,     eos_token_id=50_256							,     )
			else:
						_UpperCAmelCase	 =     GPTaConfig.from_json_file(args.config_file					)

			_UpperCAmelCase	 =     ['GPT2LMHeadModel']

			# Convert.
			print('Converting'					)
			_UpperCAmelCase	 =     convert_megatron_checkpoint(_UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase					)

			# Print the structure of converted state dict.
			if args.print_checkpoint_structure:
						recursive_print(_UpperCAmelCase							,     _UpperCAmelCase					)

			# Add tokenizer class info to config
			# see https://github.com/huggingface/transformers/issues/13906)
			if ds_args is not None:
						_UpperCAmelCase	 =     ds_args.tokenizer_type
						if tokenizer_type == "GPT2BPETokenizer":
									_UpperCAmelCase	 =     'gpt2'
						elif tokenizer_type == "PretrainedFromHF":
									_UpperCAmelCase	 =     ds_args.tokenizer_name_or_path
						else:
									raise ValueError(F"Unrecognized tokenizer_type {tokenizer_type}"					)
			else:
						_UpperCAmelCase	 =     'gpt2'

			_UpperCAmelCase	 =     AutoTokenizer.from_pretrained(_UpperCAmelCase					)
			_UpperCAmelCase	 =     type(_UpperCAmelCase					).__name__
			_UpperCAmelCase	 =     tokenizer_class

			# Store the config to file.
			print('Saving config'					)
			config.save_pretrained(_UpperCAmelCase					)

			# Save tokenizer based on args
			print(F"Adding {tokenizer_class} tokenizer files"					)
			tokenizer.save_pretrained(_UpperCAmelCase					)

			# Store the state_dict to file.
			_UpperCAmelCase	 =     os.path.join(_UpperCAmelCase							,     'pytorch_model.bin'					)
			print(F"Saving checkpoint to \"{output_checkpoint_file}\""					)
			torch.save(_UpperCAmelCase							,     _UpperCAmelCase					)


####################################################################################################

if __name__ == "__main__":
				main()

####################################################################################################





from typing import Dict, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_torch_tensor, logging


if is_torch_available():
							import torch


UpperCAmelCase__     = logging.get_logger(__name__)







class     __lowerCAmelCase					(    A		):
			UpperCamelCase           =  ['''pixel_values''']



			def __init__(		self     :						Any				,	A     :						bool = True				,	A     :						Optional[Dict[str, int]] = None				,	A     :						PILImageResampling = PILImageResampling.BILINEAR				,	A     :						bool = True				,	A     :						Dict[str, int] = None				,	A     :						bool = True				,	A     :						Union[int, float] = 1 / 2_55				,	A     :						bool = True				,	A     :						Optional[Union[float, List[float]]] = None				,	A     :						Optional[Union[float, List[float]]] = None				,	**A     :						Union[str, Any]				,	)     ->					None:



						"""simple docstring"""






						super().__init__(**A)
						_UpperCAmelCase	 =     size if size is not None else {'shortest_edge': 2_56}
						_UpperCAmelCase	 =     get_size_dict(A				,	default_to_square=A)
						_UpperCAmelCase	 =     crop_size if crop_size is not None else {'height': 2_24, 'width': 2_24}
						_UpperCAmelCase	 =     get_size_dict(A				,	param_name='crop_size')
						_UpperCAmelCase	 =     do_resize
						_UpperCAmelCase	 =     size
						_UpperCAmelCase	 =     resample
						_UpperCAmelCase	 =     do_center_crop
						_UpperCAmelCase	 =     crop_size
						_UpperCAmelCase	 =     do_rescale
						_UpperCAmelCase	 =     rescale_factor
						_UpperCAmelCase	 =     do_normalize
						_UpperCAmelCase	 =     image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
						_UpperCAmelCase	 =     image_std if image_std is not None else IMAGENET_STANDARD_STD



			def 					_lowerCamelCase (		self     :						List[str]				,	A     :						np.ndarray				,	A     :						Dict[str, int]				,	A     :						PILImageResampling = PILImageResampling.BICUBIC				,	A     :						Optional[Union[str, ChannelDimension]] = None				,	**A     :						List[str]				,	)     ->					np.ndarray:



						"""simple docstring"""






						_UpperCAmelCase	 =     get_size_dict(A				,	default_to_square=A)
						if "shortest_edge" not in size:
									raise ValueError(F"The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}")
						_UpperCAmelCase	 =     get_resize_output_image_size(A				,	size=size['shortest_edge']				,	default_to_square=A)
						return resize(A				,	size=A				,	resample=A				,	data_format=A				,	**A)



			def 					_lowerCamelCase (		self     :						Any				,	A     :						np.ndarray				,	A     :						Dict[str, int]				,	A     :						Optional[Union[str, ChannelDimension]] = None				,	**A     :						Union[str, Any]				,	)     ->					np.ndarray:



						"""simple docstring"""






						_UpperCAmelCase	 =     get_size_dict(A)
						if "height" not in size or "width" not in size:
									raise ValueError(F"The `size` parameter must contain the keys `height` and `width`. Got {size.keys()}")
						return center_crop(A				,	size=(size['height'], size['width'])				,	data_format=A				,	**A)



			def 					_lowerCamelCase (		self     :						Any				,	A     :						np.ndarray				,	A     :						float				,	A     :						Optional[Union[str, ChannelDimension]] = None				,	**A     :						Dict)     ->					np.ndarray:



						"""simple docstring"""






						return rescale(A				,	scale=A				,	data_format=A				,	**A)



			def 					_lowerCamelCase (		self     :						int				,	A     :						np.ndarray				,	A     :						Union[float, List[float]]				,	A     :						Union[float, List[float]]				,	A     :						Optional[Union[str, ChannelDimension]] = None				,	**A     :						Dict				,	)     ->					np.ndarray:



						"""simple docstring"""






						return normalize(A				,	mean=A				,	std=A				,	data_format=A				,	**A)



			def 					_lowerCamelCase (		self     :						Union[str, Any]				,	A     :						ImageInput				,	A     :						Optional[bool] = None				,	A     :						Dict[str, int] = None				,	A     :						PILImageResampling = None				,	A     :						bool = None				,	A     :						Dict[str, int] = None				,	A     :						Optional[bool] = None				,	A     :						Optional[float] = None				,	A     :						Optional[bool] = None				,	A     :						Optional[Union[float, List[float]]] = None				,	A     :						Optional[Union[float, List[float]]] = None				,	A     :						Optional[Union[str, TensorType]] = None				,	A     :						Union[str, ChannelDimension] = ChannelDimension.FIRST				,	**A     :						int				,	)     ->					Dict:



						"""simple docstring"""






						_UpperCAmelCase	 =     do_resize if do_resize is not None else self.do_resize
						_UpperCAmelCase	 =     size if size is not None else self.size
						_UpperCAmelCase	 =     get_size_dict(A				,	default_to_square=A)
						_UpperCAmelCase	 =     resample if resample is not None else self.resample
						_UpperCAmelCase	 =     do_center_crop if do_center_crop is not None else self.do_center_crop
						_UpperCAmelCase	 =     crop_size if crop_size is not None else self.crop_size
						_UpperCAmelCase	 =     get_size_dict(A				,	param_name='crop_size')
						_UpperCAmelCase	 =     do_rescale if do_rescale is not None else self.do_rescale
						_UpperCAmelCase	 =     rescale_factor if rescale_factor is not None else self.rescale_factor
						_UpperCAmelCase	 =     do_normalize if do_normalize is not None else self.do_normalize
						_UpperCAmelCase	 =     image_mean if image_mean is not None else self.image_mean
						_UpperCAmelCase	 =     image_std if image_std is not None else self.image_std

						_UpperCAmelCase	 =     make_list_of_images(A)

						if not valid_images(A):
									raise ValueError(
									    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
									    'torch.Tensor, tf.Tensor or jax.ndarray.')

						if do_resize and size is None:
									raise ValueError('Size must be specified if do_resize is True.')

						if do_center_crop and crop_size is None:
									raise ValueError('Crop size must be specified if do_center_crop is True.')

						if do_rescale and rescale_factor is None:
									raise ValueError('Rescale factor must be specified if do_rescale is True.')

						if do_normalize and (image_mean is None or image_std is None):
									raise ValueError('Image mean and std must be specified if do_normalize is True.')

						# All transformations expect numpy arrays.
						_UpperCAmelCase	 =     [to_numpy_array(A) for image in images]

						if do_resize:
									_UpperCAmelCase	 =     [self.resize(image=A				,	size=A				,	resample=A) for image in images]

						if do_center_crop:
									_UpperCAmelCase	 =     [self.center_crop(image=A				,	size=A) for image in images]

						if do_rescale:
									_UpperCAmelCase	 =     [self.rescale(image=A				,	scale=A) for image in images]

						if do_normalize:
									_UpperCAmelCase	 =     [self.normalize(image=A				,	mean=A				,	std=A) for image in images]

						_UpperCAmelCase	 =     [to_channel_dimension_format(A				,	A) for image in images]

						_UpperCAmelCase	 =     {'pixel_values': images}
						return BatchFeature(data=A				,	tensor_type=A)



			def 					_lowerCamelCase (		self     :						str				,	A     :						Any				,	A     :						List[Tuple] = None)     ->					Tuple:



						"""simple docstring"""






						_UpperCAmelCase	 =     outputs.logits

						# Resize logits and compute semantic segmentation maps
						if target_sizes is not None:
									if len(A) != len(A):
												raise ValueError(
												    'Make sure that you pass in as many target sizes as the batch dimension of the logits')

									if is_torch_tensor(A):
												_UpperCAmelCase	 =     target_sizes.numpy()

									_UpperCAmelCase	 =     []

									for idx in range(len(A)):
												_UpperCAmelCase	 =     torch.nn.functional.interpolate(
												    logits[idx].unsqueeze(dim=0)				,	size=target_sizes[idx]				,	mode='bilinear'				,	align_corners=A)
												_UpperCAmelCase	 =     resized_logits[0].argmax(dim=0)
												semantic_segmentation.append(A)
						else:
									_UpperCAmelCase	 =     logits.argmax(dim=1)
									_UpperCAmelCase	 =     [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0])]

						return semantic_segmentation
