


import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

a					  =     [
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

a					  =     [
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]


def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__  ):
					lowercase_  							=			{
					    """word_embeddings.weight""": """word_embeddings.weight""",
					    """word_embeddings.norm.weight""": """word_embeddings_layernorm.weight""",
					    """word_embeddings.norm.bias""": """word_embeddings_layernorm.bias""",
					    """weight""": """ln_f.weight""",
					    """bias""": """ln_f.bias""",
					}

					if key in layer_rename_map:
										return layer_rename_map[key]

					# Handle transformer blocks
					lowercase_  							=			int(re.match(r""".*layer_(\d*).*"""						,							UpperCAmelCase__  )[1]  )
					layer_number -= 3
					return F'''h.{layer_number}.''' + key


def 			UpperCAmelCase_				(      UpperCAmelCase__  ):
					if dtype == torch.bool:
										return 1 / 8
					lowercase_  							=			re.search(r"""[^\d](\d+)$"""						,							str(UpperCAmelCase__  )  )
					if bit_search is None:
										raise ValueError(F'''`dtype` is not a valid dtype: {dtype}.'''  )
					lowercase_  							=			int(bit_search.groups()[0]  )
					return bit_size // 8


def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  ):
					# Construct model
					if bloom_config_file == "":
										lowercase_  							=			BloomConfig()
					else:
										lowercase_  							=			BloomConfig.from_json_file(UpperCAmelCase__  )

					if shard_model:
										lowercase_  							=			os.listdir(UpperCAmelCase__  )
										lowercase_  							=			sorted(filter(lambda UpperCAmelCase__  : s.startswith("""layer"""  ) and "model_00" in s						,							UpperCAmelCase__  )  )

										lowercase_  							=			{"""weight_map""": {}, """metadata""": {}}
										lowercase_  							=			0

										lowercase_  							=			None

										lowercase_  							=			BloomConfig()

										for j, file in enumerate(UpperCAmelCase__  ):
															print("""Processing file: {}""".format(UpperCAmelCase__  )  )
															lowercase_  							=			None

															for i in range(UpperCAmelCase__  ):
																				# load all TP files
																				lowercase_  							=			file.replace("""model_00"""						,							F'''model_0{i}'''  )
																				lowercase_  							=			torch.load(os.path.join(UpperCAmelCase__						,							UpperCAmelCase__  )						,							map_location="""cpu"""  )

																				# Rename keys in the transformers names
																				lowercase_  							=			list(temp.keys()  )
																				for key in keys:
																									lowercase_  							=			temp.pop(UpperCAmelCase__  )

																				if tensors is None:
																									lowercase_  							=			temp
																				else:
																									for key in tensors.keys():
																														if any(key.endswith(UpperCAmelCase__  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																																			# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																			tensors[key] += temp[key]
																														else:
																																			# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																			lowercase_  							=			1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN  ) else 0
																																			# We concatenate these weights accross TP ranks
																																			lowercase_  							=			torch.cat([tensors[key], temp[key]]						,							dim=UpperCAmelCase__  )

            # Divide by the number of TP the weights we want to average
															for key in tensors.keys():
																				if any(key.endswith(UpperCAmelCase__  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																									lowercase_  							=			tensors[key] / pretraining_tp
															torch.save(
															    UpperCAmelCase__						,							os.path.join(
															        UpperCAmelCase__						,							"""pytorch_model_{}-of-{}.bin""".format(str(j + 1  ).zfill(5  )						,							str(len(UpperCAmelCase__  )  ).zfill(5  )  )						,							)						,							)

															for key in tensors.keys():
																				lowercase_  							=			tensors[key]
																				total_size += value.numel() * get_dtype_size(value.dtype  )
																				if key not in index_dict["weight_map"]:
																									lowercase_  							=			"""pytorch_model_{}-of-{}.bin""".format(
																									    str(j + 1  ).zfill(5  )						,							str(len(UpperCAmelCase__  )  ).zfill(5  )  )

										lowercase_  							=			BloomConfig()
										lowercase_  							=			pytorch_dump_folder_path + """/""" + CONFIG_NAME
										lowercase_  							=			total_size
										with open(UpperCAmelCase__						,							"""w"""						,							encoding="""utf-8"""  ) as f:
															f.write(config.to_json_string()  )
										with open(os.path.join(UpperCAmelCase__						,							WEIGHTS_NAME + """.index.json"""  )						,							"""w"""						,							encoding="""utf-8"""  ) as f:
															lowercase_  							=			json.dumps(UpperCAmelCase__						,							indent=2						,							sort_keys=UpperCAmelCase__  ) + """\n"""
															f.write(UpperCAmelCase__  )
					else:
										lowercase_  							=			BloomModel(UpperCAmelCase__  )

										lowercase_  							=			os.listdir(UpperCAmelCase__  )
										lowercase_  							=			sorted(filter(lambda UpperCAmelCase__  : s.startswith("""layer"""  ) and "model_00" in s						,							UpperCAmelCase__  )  )

										lowercase_  							=			None
										for i, file in enumerate(UpperCAmelCase__  ):
															lowercase_  							=			None
															for i in range(UpperCAmelCase__  ):
																				# load all TP files
																				lowercase_  							=			file.replace("""model_00"""						,							F'''model_0{i}'''  )
																				lowercase_  							=			torch.load(os.path.join(UpperCAmelCase__						,							UpperCAmelCase__  )						,							map_location="""cpu"""  )

																				# Rename keys in the transformers names
																				lowercase_  							=			list(temp.keys()  )
																				for key in keys:
																									lowercase_  							=			temp.pop(UpperCAmelCase__  )

																				if tensors is None:
																									lowercase_  							=			temp
																				else:
																									for key in tensors.keys():
																														# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																														if any(key.endswith(UpperCAmelCase__  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																																			tensors[key] += temp[key]
																														else:
																																			# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																			lowercase_  							=			1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN  ) else 0
																																			# We concatenate these weights accross TP ranks
																																			lowercase_  							=			torch.cat([tensors[key], temp[key]]						,							dim=UpperCAmelCase__  )

            # Divide by the number of TP the weights we want to average
															for key in tensors.keys():
																				if any(key.endswith(UpperCAmelCase__  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																									lowercase_  							=			tensors[key] / pretraining_tp

															lowercase_  							=			model.load_state_dict(UpperCAmelCase__						,							strict=UpperCAmelCase__  )
															assert not other_keys.unexpected_keys, F'''The keys {other_keys.unexpected_keys} are unexpected'''
															if missing_keys is None:
																				lowercase_  							=			set(other_keys.missing_keys  )
															else:
																				lowercase_  							=			missing_keys.intersection(set(other_keys.missing_keys  )  )

										assert not missing_keys, F'''The keys {missing_keys} are missing'''

										# Save pytorch-model
										os.makedirs(UpperCAmelCase__						,							exist_ok=UpperCAmelCase__  )
										lowercase_  							=			pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
										lowercase_  							=			pytorch_dump_folder_path + """/""" + CONFIG_NAME
										print(F'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''  )
										if config.torch_dtype is not None:
															lowercase_  							=			model.to(config.torch_dtype  )
										torch.save(model.state_dict()						,							UpperCAmelCase__  )
										print(F'''Save configuration file to {pytorch_config_dump_path}'''  )
										with open(UpperCAmelCase__						,							"""w"""						,							encoding="""utf-8"""  ) as f:
															f.write(config.to_json_string()  )


if __name__ == "__main__":
							a					  =     argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--bloom_checkpoint_path',
							    default=None,
							    type=str,
							    required=True,
							    help='Path to the Megatron-LM checkpoint path.',
							)
							parser.add_argument(
							    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
							)
							parser.add_argument(
							    '--bloom_config_file',
							    default='',
							    type=str,
							    help=(
							        'An optional config json file corresponding to the pre-trained model. \n'
							        'This specifies the model architecture.'
							    ),
							)
							parser.add_argument(
							    '--shard_model',
							    action='store_true',
							    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
							)
							parser.add_argument(
							    '--pretraining_tp',
							    default=4,
							    type=int,
							    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
							)
							a					  =     parser.parse_args()
							convert_bloom_checkpoint_to_pytorch(
							    args.bloom_checkpoint_path,
							    args.bloom_config_file,
							    args.pytorch_dump_folder_path,
							    args.shard_model,
							    args.pretraining_tp,
							)






from typing import Dict, Iterable, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


a					  =     logging.get_logger(__name__)







class        UpperCamelCase__   (	__magic_name__   ):
					__SCREAMING_SNAKE_CASE       :  List[Any]						=   ['pixel_values']

					def __init__(     self       :   List[str]							,					UpperCamelCase__       :   bool = True							,					UpperCamelCase__       :   Dict[str, int] = None							,					UpperCamelCase__       :   PILImageResampling = PILImageResampling.BICUBIC							,					UpperCamelCase__       :   bool = True							,					UpperCamelCase__       :   Dict[str, int] = None							,					UpperCamelCase__       :   bool = True							,					UpperCamelCase__       :   Union[int, float] = 1 / 255							,					UpperCamelCase__       :   bool = True							,					UpperCamelCase__       :   Optional[Union[float, Iterable[float]]] = IMAGENET_DEFAULT_MEAN							,					UpperCamelCase__       :   Optional[Union[float, Iterable[float]]] = IMAGENET_DEFAULT_STD							,					**UpperCamelCase__       :   Dict							,					):






										'''simple docstring'''






										super().__init__(**UpperCamelCase__       )
										lowercase_  							=			size if size is not None else {"""shortest_edge""": 224}
										lowercase_  							=			get_size_dict(UpperCamelCase__							,					default_to_square=UpperCamelCase__       )
										lowercase_  							=			crop_size if crop_size is not None else {"""height""": 224, """width""": 224}
										lowercase_  							=			get_size_dict(UpperCamelCase__							,					param_name="""crop_size"""       )

										lowercase_  							=			do_resize
										lowercase_  							=			size
										lowercase_  							=			resample
										lowercase_  							=			do_center_crop
										lowercase_  							=			crop_size
										lowercase_  							=			do_rescale
										lowercase_  							=			rescale_factor
										lowercase_  							=			do_normalize
										lowercase_  							=			image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
										lowercase_  							=			image_std if image_std is not None else IMAGENET_DEFAULT_STD

					def   UpperCAmelCase__       (     self       :   int							,					UpperCamelCase__       :   np.ndarray							,					UpperCamelCase__       :   Dict[str, int]							,					UpperCamelCase__       :   PILImageResampling = PILImageResampling.BICUBIC							,					UpperCamelCase__       :   Optional[Union[str, ChannelDimension]] = None							,					**UpperCamelCase__       :   Optional[int]							,					):






										'''simple docstring'''






										lowercase_  							=			get_size_dict(UpperCamelCase__							,					default_to_square=UpperCamelCase__       )
										# size_dict is a dict with either keys "height" and "width" or "shortest_edge"
										if "shortest_edge" in size:
															lowercase_  							=			int((256 / 224) * size["""shortest_edge"""]       )
															lowercase_  							=			get_resize_output_image_size(UpperCamelCase__							,					size=UpperCamelCase__							,					default_to_square=UpperCamelCase__       )
															lowercase_  							=			{"""height""": output_size[0], """width""": output_size[1]}
										if "height" not in size_dict or "width" not in size_dict:
															raise ValueError(
															    F'''Size dict must have keys \'height\' and \'width\' or \'shortest_edge\'. Got {size_dict.keys()}'''       )
										return resize(
										    UpperCamelCase__							,					size=(size_dict["""height"""], size_dict["""width"""])							,					resample=UpperCamelCase__							,					data_format=UpperCamelCase__							,					**UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   str							,					UpperCamelCase__       :   np.ndarray							,					UpperCamelCase__       :   Dict[str, int]							,					UpperCamelCase__       :   Optional[Union[str, ChannelDimension]] = None							,					**UpperCamelCase__       :   str							,					):






										'''simple docstring'''






										lowercase_  							=			get_size_dict(UpperCamelCase__       )
										if "height" not in size or "width" not in size:
															raise ValueError(F'''Size dict must have keys \'height\' and \'width\'. Got {size.keys()}'''       )
										return center_crop(UpperCamelCase__							,					size=(size["""height"""], size["""width"""])							,					data_format=UpperCamelCase__							,					**UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   Optional[int]							,					UpperCamelCase__       :   np.ndarray							,					UpperCamelCase__       :   Union[int, float]							,					UpperCamelCase__       :   Optional[Union[str, ChannelDimension]] = None							,					**UpperCamelCase__       :   str							,					):






										'''simple docstring'''






										return rescale(UpperCamelCase__							,					scale=UpperCamelCase__							,					data_format=UpperCamelCase__							,					**UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   List[str]							,					UpperCamelCase__       :   np.ndarray							,					UpperCamelCase__       :   Union[float, List[float]]							,					UpperCamelCase__       :   Union[float, List[float]]							,					UpperCamelCase__       :   Optional[Union[str, ChannelDimension]] = None							,					**UpperCamelCase__       :   Optional[int]							,					):






										'''simple docstring'''






										return normalize(UpperCamelCase__							,					mean=UpperCamelCase__							,					std=UpperCamelCase__							,					data_format=UpperCamelCase__							,					**UpperCamelCase__       )







					def   UpperCAmelCase__       (     self       :   Dict							,					UpperCamelCase__       :   ImageInput							,					UpperCamelCase__       :   Optional[bool] = None							,					UpperCamelCase__       :   Optional[Dict[str, int]] = None							,					UpperCamelCase__       :   PILImageResampling = None							,					UpperCamelCase__       :   Optional[bool] = None							,					UpperCamelCase__       :   Optional[Dict[str, int]] = None							,					UpperCamelCase__       :   Optional[bool] = None							,					UpperCamelCase__       :   Optional[float] = None							,					UpperCamelCase__       :   Optional[bool] = None							,					UpperCamelCase__       :   Optional[Union[float, Iterable[float]]] = None							,					UpperCamelCase__       :   Optional[Union[float, Iterable[float]]] = None							,					UpperCamelCase__       :   Optional[TensorType] = None							,					UpperCamelCase__       :   ChannelDimension = ChannelDimension.FIRST							,					**UpperCamelCase__       :   Optional[int]							,					):






										'''simple docstring'''






										lowercase_  							=			do_resize if do_resize is not None else self.do_resize
										lowercase_  							=			resample if resample is not None else self.resample
										lowercase_  							=			do_center_crop if do_center_crop is not None else self.do_center_crop
										lowercase_  							=			do_rescale if do_rescale is not None else self.do_rescale
										lowercase_  							=			rescale_factor if rescale_factor is not None else self.rescale_factor
										lowercase_  							=			do_normalize if do_normalize is not None else self.do_normalize
										lowercase_  							=			image_mean if image_mean is not None else self.image_mean
										lowercase_  							=			image_std if image_std is not None else self.image_std

										lowercase_  							=			size if size is not None else self.size
										lowercase_  							=			get_size_dict(UpperCamelCase__							,					default_to_square=UpperCamelCase__       )
										lowercase_  							=			crop_size if crop_size is not None else self.crop_size
										lowercase_  							=			get_size_dict(UpperCamelCase__							,					param_name="""crop_size"""       )

										lowercase_  							=			make_list_of_images(UpperCamelCase__       )

										if not valid_images(UpperCamelCase__       ):
															raise ValueError(
															    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
															    """torch.Tensor, tf.Tensor or jax.ndarray."""       )

										if do_resize and size is None:
															raise ValueError("""Size must be specified if do_resize is True."""       )

										if do_center_crop and crop_size is None:
															raise ValueError("""Crop size must be specified if do_center_crop is True."""       )

										if do_rescale and rescale_factor is None:
															raise ValueError("""Rescale factor must be specified if do_rescale is True."""       )

										if do_normalize and (image_mean is None or image_std is None):
															raise ValueError("""Image mean and std must be specified if do_normalize is True."""       )

										# All transformations expect numpy arrays.
										lowercase_  							=			[to_numpy_array(UpperCamelCase__       ) for image in images]

										if do_resize:
															lowercase_  							=			[self.resize(UpperCamelCase__							,					UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										if do_center_crop:
															lowercase_  							=			[self.center_crop(UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										if do_rescale:
															lowercase_  							=			[self.rescale(UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										if do_normalize:
															lowercase_  							=			[self.normalize(UpperCamelCase__							,					UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										lowercase_  							=			[to_channel_dimension_format(UpperCamelCase__							,					UpperCamelCase__       ) for image in images]

										lowercase_  							=			{"""pixel_values""": images}
										return BatchFeature(data=UpperCamelCase__							,					tensor_type=UpperCamelCase__       )


