



# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig





def 							A			(  _UpperCAmelCase       :     Any							,     _UpperCAmelCase       :     Union[str, Any]							,     _UpperCAmelCase       :     Tuple=0					)       ->				Optional[Any]:







			'''simple docstring'''
			# Format the message.
			if name is None:
						_UpperCAmelCase	 =     None
			else:
						_UpperCAmelCase	 =     '.' * max(0							,     spaces - 2					) + '# {:' + str(50 - spaces					) + 's}'
						_UpperCAmelCase	 =     fmt.format(_UpperCAmelCase					)

			# Print and recurse (if needed).
			if isinstance(_UpperCAmelCase							,     _UpperCAmelCase					):
						if msg is not None:
									print(_UpperCAmelCase					)
						for k in val.keys():
									recursive_print(_UpperCAmelCase							,     val[k]							,     spaces + 2					)
			elif isinstance(_UpperCAmelCase							,     torch.Tensor					):
						print(_UpperCAmelCase							,     ':'							,     val.size()					)
			else:
						print(_UpperCAmelCase							,     ':'							,     _UpperCAmelCase					)





def 							A			(  _UpperCAmelCase       :     Union[str, Any]							,     _UpperCAmelCase       :     Dict							,     _UpperCAmelCase       :     Optional[int]							,     _UpperCAmelCase       :     Tuple							,     _UpperCAmelCase       :     Union[str, Any]					)       ->				Any:







			'''simple docstring'''
			# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
			# for compatibility with later versions of NVIDIA Megatron-LM.
			# The inverse operation is performed inside Megatron-LM to read checkpoints:
			# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
			# If param is the weight tensor of the self-attention block, the returned tensor
			# will have to be transposed one more time to be read by HuggingFace GPT2.
			_UpperCAmelCase	 =     param.size()
			if checkpoint_version == 1.0:
						# version 1.0 stores [num_heads * hidden_size * num_splits, :]
						_UpperCAmelCase	 =     (num_heads, hidden_size, num_splits) + input_shape[1:]
						_UpperCAmelCase	 =     param.view(*_UpperCAmelCase					)
						_UpperCAmelCase	 =     param.transpose(0							,     2					)
						_UpperCAmelCase	 =     param.transpose(1							,     2					).contiguous()
			elif checkpoint_version >= 2.0:
						# other versions store [num_heads * num_splits * hidden_size, :]
						_UpperCAmelCase	 =     (num_heads, num_splits, hidden_size) + input_shape[1:]
						_UpperCAmelCase	 =     param.view(*_UpperCAmelCase					)
						_UpperCAmelCase	 =     param.transpose(0							,     1					).contiguous()
			_UpperCAmelCase	 =     param.view(*_UpperCAmelCase					)
			return param





def 							A			(  _UpperCAmelCase       :     Optional[Any]							,     _UpperCAmelCase       :     List[str]							,     _UpperCAmelCase       :     Dict					)       ->				Optional[Any]:







			'''simple docstring'''
			# The converted output model.
			_UpperCAmelCase	 =     {}

			# old versions did not store training args
			_UpperCAmelCase	 =     input_state_dict.get('args'							,     _UpperCAmelCase					)
			if ds_args is not None:
						# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
						# from pprint import pprint
						# pprint(vars(ds_args))

						_UpperCAmelCase	 =     ds_args.padded_vocab_size
						_UpperCAmelCase	 =     ds_args.max_position_embeddings
						_UpperCAmelCase	 =     ds_args.hidden_size
						_UpperCAmelCase	 =     ds_args.num_layers
						_UpperCAmelCase	 =     ds_args.num_attention_heads
						_UpperCAmelCase	 =     ds_args.ffn_hidden_size
						# pprint(config)

			# The number of heads.
			_UpperCAmelCase	 =     config.n_head
			# The hidden_size per head.
			_UpperCAmelCase	 =     config.n_embd // config.n_head
			# Megatron-LM checkpoint version
			if "checkpoint_version" in input_state_dict.keys():
						_UpperCAmelCase	 =     input_state_dict['checkpoint_version']
			else:
						_UpperCAmelCase	 =     0.0

			# The model.
			_UpperCAmelCase	 =     input_state_dict['model']
			# The language model.
			_UpperCAmelCase	 =     model['language_model']
			# The embeddings.
			_UpperCAmelCase	 =     lm['embedding']

			# The word embeddings.
			_UpperCAmelCase	 =     embeddings['word_embeddings']['weight']
			# Truncate the embedding table to vocab_size rows.
			_UpperCAmelCase	 =     word_embeddings[: config.vocab_size, :]
			_UpperCAmelCase	 =     word_embeddings

			# The position embeddings.
			_UpperCAmelCase	 =     embeddings['position_embeddings']['weight']
			# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
			_UpperCAmelCase	 =     pos_embeddings.size(0					)
			if n_positions != config.n_positions:
						raise ValueError(
						    F"pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"					)
			# Store the position embeddings.
			_UpperCAmelCase	 =     pos_embeddings

			# The transformer.
			_UpperCAmelCase	 =     lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

			# The regex to extract layer names.
			_UpperCAmelCase	 =     re.compile(R'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'					)

			# The simple map of names for "automated" rules.
			_UpperCAmelCase	 =     {
			    'attention.dense': '.attn.c_proj.',
			    'self_attention.dense': '.attn.c_proj.',
			    'mlp.dense_h_to_4h': '.mlp.c_fc.',
			    'mlp.dense_4h_to_h': '.mlp.c_proj.',
			}

			# Extract the layers.
			for key, val in transformer.items():
						# Match the name.
						_UpperCAmelCase	 =     layer_re.match(_UpperCAmelCase					)

						# Stop if that's not a layer
						if m is None:
									break

						# The index of the layer.
						_UpperCAmelCase	 =     int(m.group(1					)					)
						# The name of the operation.
						_UpperCAmelCase	 =     m.group(2					)
						# Is it a weight or a bias?
						_UpperCAmelCase	 =     m.group(3					)

						# The name of the layer.
						_UpperCAmelCase	 =     F"transformer.h.{layer_idx}"

						# For layernorm(s), simply store the layer norm.
						if op_name.endswith('layernorm'					):
									_UpperCAmelCase	 =     'ln_1' if op_name.startswith('input'					) else 'ln_2'
									_UpperCAmelCase	 =     val

						# Transpose the QKV matrix.
						elif (
						    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
						) and weight_or_bias == "weight":
									# Insert a tensor of 1x1xDxD bias.
									_UpperCAmelCase	 =     torch.tril(torch.ones((n_positions, n_positions)							,     dtype=torch.floataa					)					).view(
									    1							,     1							,     _UpperCAmelCase							,     _UpperCAmelCase					)
									_UpperCAmelCase	 =     causal_mask

									# Insert a "dummy" tensor for masked_bias.
									_UpperCAmelCase	 =     torch.tensor(-1E4							,     dtype=torch.floataa					)
									_UpperCAmelCase	 =     masked_bias

									_UpperCAmelCase	 =     fix_query_key_value_ordering(_UpperCAmelCase							,     _UpperCAmelCase							,     3							,     _UpperCAmelCase							,     _UpperCAmelCase					)
									# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
									_UpperCAmelCase	 =     out_val.transpose(0							,     1					).contiguous()
									# Store.
									_UpperCAmelCase	 =     out_val

						# Transpose the bias.
						elif (
						    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
						) and weight_or_bias == "bias":
									_UpperCAmelCase	 =     fix_query_key_value_ordering(_UpperCAmelCase							,     _UpperCAmelCase							,     3							,     _UpperCAmelCase							,     _UpperCAmelCase					)
									# Store. No change of shape.
									_UpperCAmelCase	 =     out_val

						# Transpose the weights.
						elif weight_or_bias == "weight":
									_UpperCAmelCase	 =     megatron_to_transformers[op_name]
									_UpperCAmelCase	 =     val.transpose(0							,     1					)

						# Copy the bias.
						elif weight_or_bias == "bias":
									_UpperCAmelCase	 =     megatron_to_transformers[op_name]
									_UpperCAmelCase	 =     val

    # DEBUG.
			assert config.n_layer == layer_idx + 1

			# The final layernorm.
			_UpperCAmelCase	 =     transformer['final_layernorm.weight']
			_UpperCAmelCase	 =     transformer['final_layernorm.bias']

			# For LM head, transformers' wants the matrix to weight embeddings.
			_UpperCAmelCase	 =     word_embeddings

			# It should be done!
			return output_state_dict





def 							A			(  )       ->				Dict:







			'''simple docstring'''
			# Create the argument parser.
			_UpperCAmelCase	 =     argparse.ArgumentParser()
			parser.add_argument('--print-checkpoint-structure'							,     action='store_true'					)
			parser.add_argument(
			    'path_to_checkpoint'							,     type=_UpperCAmelCase							,     help='Path to the checkpoint file (.zip archive or direct .pt file)'							,     )
			parser.add_argument(
			    '--config_file'							,     default=''							,     type=_UpperCAmelCase							,     help='An optional config json file describing the pre-trained model.'							,     )
			_UpperCAmelCase	 =     parser.parse_args()

			# Extract the basename.
			_UpperCAmelCase	 =     os.path.dirname(args.path_to_checkpoint					)

			# Load the model.
			# the .zip is very optional, let's keep it for backward compatibility
			print(F"Extracting PyTorch state dictionary from {args.path_to_checkpoint}"					)
			if args.path_to_checkpoint.endswith('.zip'					):
						with zipfile.ZipFile(args.path_to_checkpoint							,     'r'					) as checkpoint:
									with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'					) as pytorch_dict:
												_UpperCAmelCase	 =     torch.load(_UpperCAmelCase							,     map_location='cpu'					)
			else:
						_UpperCAmelCase	 =     torch.load(args.path_to_checkpoint							,     map_location='cpu'					)

			_UpperCAmelCase	 =     input_state_dict.get('args'							,     _UpperCAmelCase					)

			# Read the config, or default to the model released by NVIDIA.
			if args.config_file == "":
						if ds_args is not None:
									if ds_args.bias_gelu_fusion:
												_UpperCAmelCase	 =     'gelu_fast'
									elif ds_args.openai_gelu:
												_UpperCAmelCase	 =     'gelu_new'
									else:
												_UpperCAmelCase	 =     'gelu'
						else:
									# in the very early days this used to be "gelu_new"
									_UpperCAmelCase	 =     'gelu_new'

						# Spell out all parameters in case the defaults change.
						_UpperCAmelCase	 =     GPTaConfig(
						    vocab_size=50_257							,     n_positions=1_024							,     n_embd=1_024							,     n_layer=24							,     n_head=16							,     n_inner=4_096							,     activation_function=_UpperCAmelCase							,     resid_pdrop=0.1							,     embd_pdrop=0.1							,     attn_pdrop=0.1							,     layer_norm_epsilon=1E-5							,     initializer_range=0.02							,     summary_type='cls_index'							,     summary_use_proj=_UpperCAmelCase							,     summary_activation=_UpperCAmelCase							,     summary_proj_to_labels=_UpperCAmelCase							,     summary_first_dropout=0.1							,     scale_attn_weights=_UpperCAmelCase							,     use_cache=_UpperCAmelCase							,     bos_token_id=50_256							,     eos_token_id=50_256							,     )
			else:
						_UpperCAmelCase	 =     GPTaConfig.from_json_file(args.config_file					)

			_UpperCAmelCase	 =     ['GPT2LMHeadModel']

			# Convert.
			print('Converting'					)
			_UpperCAmelCase	 =     convert_megatron_checkpoint(_UpperCAmelCase							,     _UpperCAmelCase							,     _UpperCAmelCase					)

			# Print the structure of converted state dict.
			if args.print_checkpoint_structure:
						recursive_print(_UpperCAmelCase							,     _UpperCAmelCase					)

			# Add tokenizer class info to config
			# see https://github.com/huggingface/transformers/issues/13906)
			if ds_args is not None:
						_UpperCAmelCase	 =     ds_args.tokenizer_type
						if tokenizer_type == "GPT2BPETokenizer":
									_UpperCAmelCase	 =     'gpt2'
						elif tokenizer_type == "PretrainedFromHF":
									_UpperCAmelCase	 =     ds_args.tokenizer_name_or_path
						else:
									raise ValueError(F"Unrecognized tokenizer_type {tokenizer_type}"					)
			else:
						_UpperCAmelCase	 =     'gpt2'

			_UpperCAmelCase	 =     AutoTokenizer.from_pretrained(_UpperCAmelCase					)
			_UpperCAmelCase	 =     type(_UpperCAmelCase					).__name__
			_UpperCAmelCase	 =     tokenizer_class

			# Store the config to file.
			print('Saving config'					)
			config.save_pretrained(_UpperCAmelCase					)

			# Save tokenizer based on args
			print(F"Adding {tokenizer_class} tokenizer files"					)
			tokenizer.save_pretrained(_UpperCAmelCase					)

			# Store the state_dict to file.
			_UpperCAmelCase	 =     os.path.join(_UpperCAmelCase							,     'pytorch_model.bin'					)
			print(F"Saving checkpoint to \"{output_checkpoint_file}\""					)
			torch.save(_UpperCAmelCase							,     _UpperCAmelCase					)


####################################################################################################

if __name__ == "__main__":
							main()

####################################################################################################





import random
import unittest

import torch

from diffusers import IFInpaintingPipeline
from diffusers.utils import floats_tensor
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import skip_mps, torch_device

from ..pipeline_params import (
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_PARAMS,
)
from ..test_pipelines_common import PipelineTesterMixin
from . import IFPipelineTesterMixin







@skip_mps
class     __lowerCAmelCase					(    A       ,		A       ,		unittest.TestCase		):
			UpperCamelCase           =  IFInpaintingPipeline
			UpperCamelCase           =  TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {'''width''', '''height'''}
			UpperCamelCase           =  TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS
			UpperCamelCase           =  PipelineTesterMixin.required_optional_params - {'''latents'''}



			def 					_lowerCamelCase (		self     :						List[str])     ->					Tuple:



						"""simple docstring"""






						return self._get_dummy_components()



			def 					_lowerCamelCase (		self     :						Any				,	A     :						int				,	A     :						Dict=0)     ->					Tuple:



						"""simple docstring"""






						if str(A).startswith('mps'):
									_UpperCAmelCase	 =     torch.manual_seed(A)
						else:
									_UpperCAmelCase	 =     torch.Generator(device=A).manual_seed(A)

						_UpperCAmelCase	 =     floats_tensor((1, 3, 32, 32)				,	rng=random.Random(A)).to(A)
						_UpperCAmelCase	 =     floats_tensor((1, 3, 32, 32)				,	rng=random.Random(A)).to(A)

						_UpperCAmelCase	 =     {
						    'prompt': 'A painting of a squirrel eating a burger',
						    'image': image,
						    'mask_image': mask_image,
						    'generator': generator,
						    'num_inference_steps': 2,
						    'output_type': 'numpy',
						}

						return inputs



			@unittest.skipIf(
			    torch_device != 'cuda' or not is_xformers_available()				,	reason='XFormers attention is only available with CUDA and `xformers` installed'				,	)
			def 					_lowerCamelCase (		self     :						List[str])     ->					Union[str, Any]:



						"""simple docstring"""






						self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1E-3)



			def 					_lowerCamelCase (		self     :						Optional[Any])     ->					Optional[int]:



						"""simple docstring"""






						self._test_save_load_optional_components()



			@unittest.skipIf(torch_device != 'cuda'				,	reason='float16 requires CUDA')
			def 					_lowerCamelCase (		self     :						List[str])     ->					Any:



						"""simple docstring"""






						super().test_save_load_floataa(expected_max_diff=1E-1)



			def 					_lowerCamelCase (		self     :						Optional[int])     ->					Tuple:



						"""simple docstring"""






						self._test_attention_slicing_forward_pass(expected_max_diff=1E-2)



			def 					_lowerCamelCase (		self     :						str)     ->					List[str]:



						"""simple docstring"""






						self._test_save_load_local()



			def 					_lowerCamelCase (		self     :						int)     ->					Tuple:



						"""simple docstring"""






						self._test_inference_batch_single_identical(
						    expected_max_diff=1E-2				,	)
