



"""simple docstring"""






import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_lowercase				:				int   		=		[
    """word_embeddings_layernorm.weight""",
    """word_embeddings_layernorm.bias""",
    """input_layernorm.weight""",
    """input_layernorm.bias""",
    """post_attention_layernorm.weight""",
    """post_attention_layernorm.bias""",
    """self_attention.dense.bias""",
    """mlp.dense_4h_to_h.bias""",
    """ln_f.weight""",
    """ln_f.bias""",
]

_lowercase				:				Any   		=		[
    """mlp.dense_4h_to_h.weight""",
    """self_attention.dense.weight""",
]






def   snake_case__     (	__lowerCamelCase       :					Dict							,	__lowerCamelCase       :					Optional[int]			):
			"""simple docstring"""
			lowerCamelCase__	:	int				={
			    '''word_embeddings.weight''': '''word_embeddings.weight''',
			    '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
			    '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
			    '''weight''': '''ln_f.weight''',
			    '''bias''': '''ln_f.bias''',
			}

			if key in layer_rename_map:
						return layer_rename_map[key]

			# Handle transformer blocks
			lowerCamelCase__	:	Dict				=int(re.match(R'''.*layer_(\d*).*'''							,	__lowerCamelCase			)[1]			)
			layer_number -= 3
			return f'''h.{layer_number}.''' + key






def   snake_case__     (	__lowerCamelCase       :					Dict			):
			"""simple docstring"""






			if dtype == torch.bool:
						return 1 / 8
			lowerCamelCase__	:	Tuple				=re.search(R'''[^\d](\d+)$'''							,	str(__lowerCamelCase			)			)
			if bit_search is None:
						raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.'''			)
			lowerCamelCase__	:	Union[str, Any]				=int(bit_search.groups()[0]			)
			return bit_size // 8






def   snake_case__     (	__lowerCamelCase       :					Union[str, Any]							,	__lowerCamelCase       :					str							,	__lowerCamelCase       :					Any							,	__lowerCamelCase       :					List[Any]							,	__lowerCamelCase       :					str			):
			"""simple docstring"""






			if bloom_config_file == "":
						lowerCamelCase__	:	Tuple				=BloomConfig()
			else:
						lowerCamelCase__	:	Tuple				=BloomConfig.from_json_file(__lowerCamelCase			)

			if shard_model:
						lowerCamelCase__	:	List[str]				=os.listdir(__lowerCamelCase			)
						lowerCamelCase__	:	List[str]				=sorted(filter(lambda __lowerCamelCase			: s.startswith('''layer'''			) and "model_00" in s							,	__lowerCamelCase			)			)

						lowerCamelCase__	:	Any				={'''weight_map''': {}, '''metadata''': {}}
						lowerCamelCase__	:	Any				=0

						lowerCamelCase__	:	Union[str, Any]				=None

						lowerCamelCase__	:	Tuple				=BloomConfig()

						for j, file in enumerate(__lowerCamelCase			):
									print('''Processing file: {}'''.format(__lowerCamelCase			)			)
									lowerCamelCase__	:	Any				=None

									for i in range(__lowerCamelCase			):
												# load all TP files
												lowerCamelCase__	:	Dict				=file.replace('''model_00'''							,	f'''model_0{i}'''			)
												lowerCamelCase__	:	List[str]				=torch.load(os.path.join(__lowerCamelCase							,	__lowerCamelCase			)							,	map_location='''cpu'''			)

												# Rename keys in the transformers names
												lowerCamelCase__	:	List[str]				=list(temp.keys()			)
												for key in keys:
															lowerCamelCase__	:	Dict				=temp.pop(__lowerCamelCase			)

												if tensors is None:
															lowerCamelCase__	:	List[str]				=temp
												else:
															for key in tensors.keys():
																		if any(key.endswith(__lowerCamelCase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
																					# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					lowerCamelCase__	:	int				=1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN			) else 0
																					# We concatenate these weights accross TP ranks
																					lowerCamelCase__	:	Tuple				=torch.cat([tensors[key], temp[key]]							,	dim=__lowerCamelCase			)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(__lowerCamelCase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
															lowerCamelCase__	:	Tuple				=tensors[key] / pretraining_tp
									torch.save(
									    __lowerCamelCase							,	os.path.join(
									        __lowerCamelCase							,	'''pytorch_model_{}-of-{}.bin'''.format(str(j + 1			).zfill(5			)							,	str(len(__lowerCamelCase			)			).zfill(5			)			)							,	)							,	)

									for key in tensors.keys():
												lowerCamelCase__	:	Any				=tensors[key]
												total_size += value.numel() * get_dtype_size(value.dtype			)
												if key not in index_dict["weight_map"]:
															lowerCamelCase__	:	List[str]				='''pytorch_model_{}-of-{}.bin'''.format(
															    str(j + 1			).zfill(5			)							,	str(len(__lowerCamelCase			)			).zfill(5			)			)

						lowerCamelCase__	:	Dict				=BloomConfig()
						lowerCamelCase__	:	int				=pytorch_dump_folder_path + '''/''' + CONFIG_NAME
						lowerCamelCase__	:	str				=total_size
						with open(__lowerCamelCase							,	'''w'''							,	encoding='''utf-8'''			) as f:
									f.write(config.to_json_string()			)
						with open(os.path.join(__lowerCamelCase							,	WEIGHTS_NAME + '''.index.json'''			)							,	'''w'''							,	encoding='''utf-8'''			) as f:
									lowerCamelCase__	:	Tuple				=json.dumps(__lowerCamelCase							,	indent=2							,	sort_keys=__lowerCamelCase			) + '''\n'''
									f.write(__lowerCamelCase			)
			else:
						lowerCamelCase__	:	Optional[int]				=BloomModel(__lowerCamelCase			)

						lowerCamelCase__	:	List[str]				=os.listdir(__lowerCamelCase			)
						lowerCamelCase__	:	Optional[Any]				=sorted(filter(lambda __lowerCamelCase			: s.startswith('''layer'''			) and "model_00" in s							,	__lowerCamelCase			)			)

						lowerCamelCase__	:	Dict				=None
						for i, file in enumerate(__lowerCamelCase			):
									lowerCamelCase__	:	List[Any]				=None
									for i in range(__lowerCamelCase			):
												# load all TP files
												lowerCamelCase__	:	Optional[int]				=file.replace('''model_00'''							,	f'''model_0{i}'''			)
												lowerCamelCase__	:	Dict				=torch.load(os.path.join(__lowerCamelCase							,	__lowerCamelCase			)							,	map_location='''cpu'''			)

												# Rename keys in the transformers names
												lowerCamelCase__	:	Union[str, Any]				=list(temp.keys()			)
												for key in keys:
															lowerCamelCase__	:	Any				=temp.pop(__lowerCamelCase			)

												if tensors is None:
															lowerCamelCase__	:	List[str]				=temp
												else:
															for key in tensors.keys():
																		# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																		if any(key.endswith(__lowerCamelCase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					lowerCamelCase__	:	Dict				=1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN			) else 0
																					# We concatenate these weights accross TP ranks
																					lowerCamelCase__	:	Optional[Any]				=torch.cat([tensors[key], temp[key]]							,	dim=__lowerCamelCase			)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(__lowerCamelCase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
															lowerCamelCase__	:	List[str]				=tensors[key] / pretraining_tp

									lowerCamelCase__	:	Dict				=model.load_state_dict(__lowerCamelCase							,	strict=__lowerCamelCase			)
									assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
									if missing_keys is None:
												lowerCamelCase__	:	Optional[int]				=set(other_keys.missing_keys			)
									else:
												lowerCamelCase__	:	Tuple				=missing_keys.intersection(set(other_keys.missing_keys			)			)

						assert not missing_keys, f'''The keys {missing_keys} are missing'''

						# Save pytorch-model
						os.makedirs(__lowerCamelCase							,	exist_ok=__lowerCamelCase			)
						lowerCamelCase__	:	Optional[int]				=pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
						lowerCamelCase__	:	Optional[int]				=pytorch_dump_folder_path + '''/''' + CONFIG_NAME
						print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''			)
						if config.torch_dtype is not None:
									lowerCamelCase__	:	Union[str, Any]				=model.to(config.torch_dtype			)
						torch.save(model.state_dict()							,	__lowerCamelCase			)
						print(f'''Save configuration file to {pytorch_config_dump_path}'''			)
						with open(__lowerCamelCase							,	'''w'''							,	encoding='''utf-8'''			) as f:
									f.write(config.to_json_string()			)


if __name__ == "__main__":
		_lowercase				:				int   		=		argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    "--bloom_checkpoint_path",
		    default=None,
		    type=str,
		    required=True,
		    help="Path to the Megatron-LM checkpoint path.",
		)
		parser.add_argument(
		    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
		)
		parser.add_argument(
		    "--bloom_config_file",
		    default="",
		    type=str,
		    help=(
		        "An optional config json file corresponding to the pre-trained model. \n"
		        "This specifies the model architecture."
		    ),
		)
		parser.add_argument(
		    "--shard_model",
		    action="store_true",
		    help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
		)
		parser.add_argument(
		    "--pretraining_tp",
		    default=4,
		    type=int,
		    help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
		)
		_lowercase				:				List[Any]   		=		parser.parse_args()
		convert_bloom_checkpoint_to_pytorch(
		    args.bloom_checkpoint_path,
		    args.bloom_config_file,
		    args.pytorch_dump_folder_path,
		    args.shard_model,
		    args.pretraining_tp,
		)









"""simple docstring"""






import os






def   snake_case__     (	):
			"""simple docstring"""
			with open(os.path.dirname(__lowerCamelCase			) + '''/p022_names.txt'''			) as file:
						lowerCamelCase__	:	Tuple				=str(file.readlines()[0]			)
						lowerCamelCase__	:	int				=names.replace('''"'''							,	''''''			).split(''','''			)

			names.sort()

			lowerCamelCase__	:	Union[str, Any]				=0
			lowerCamelCase__	:	str				=0

			for i, name in enumerate(__lowerCamelCase			):
						for letter in name:
									name_score += ord(__lowerCamelCase			) - 64

						total_score += (i + 1) * name_score
						lowerCamelCase__	:	Dict				=0
			return total_score


if __name__ == "__main__":
		print(solution())




