
'''simple docstring'''


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
						from transformers import LlamaTokenizerFast
except ImportError as e:
						warnings.warn(e)
						warnings.warn(
						    '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
						)
						lowercase_      :				List[Any]   					=    None
lowercase_      :				Any   					=    {
    '''7B''': 1_1008,
    '''13B''': 1_3824,
    '''30B''': 1_7920,
    '''65B''': 2_2016,
    '''70B''': 2_8672,
}
lowercase_      :				str   					=    {
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      Optional[int] ,  lowercase_    :      Any=1 ,  lowercase_    :      Optional[int]=256						):
						return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3						)						) + multiple_of - 1) // multiple_of)







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      Tuple						):
						with open(lowercase_ ,  """r"""						) as f:
												return json.load(lowercase_						)







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      List[str] ,  lowercase_    :      int						):
						with open(lowercase_ ,  """w"""						) as f:
												json.dump(lowercase_ ,  lowercase_						)







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      List[str] ,  lowercase_    :      Union[str, Any] ,  lowercase_    :      int ,  lowercase_    :      str=True						):
						os.makedirs(lowercase_ ,  exist_ok=lowercase_						)
						lowercase        =							os.path.join(lowercase_ ,  """tmp"""						)
						os.makedirs(lowercase_ ,  exist_ok=lowercase_						)

						lowercase        =							read_json(os.path.join(lowercase_ ,  """params.json"""						)						)
						lowercase        =							NUM_SHARDS[model_size]
						lowercase        =							params["""n_layers"""]
						lowercase        =							params["""n_heads"""]
						lowercase        =							n_heads // num_shards
						lowercase        =							params["""dim"""]
						lowercase        =							dim // n_heads
						lowercase        =							1_0000.0
						lowercase        =							1.0 / (base ** (torch.arange(0 ,  lowercase_ ,  2						).float() / dims_per_head))

						if "n_kv_heads" in params:
												lowercase        =							params["""n_kv_heads"""]  # for GQA / MQA
												lowercase        =							n_heads_per_shard // num_key_value_heads
												lowercase        =							dim // num_key_value_heads
						else:  # compatibility with other checkpoints
												lowercase        =							n_heads
												lowercase        =							n_heads_per_shard
												lowercase        =							dim

						# permute for sliced rotary
						def permute(lowercase_    :      Tuple ,  lowercase_    :      List[Any]=n_heads ,  lowercase_    :      int=dim ,  lowercase_    :      List[str]=dim						):
												return w.view(lowercase_ ,  dima // n_heads // 2 ,  2 ,  lowercase_						).transpose(1 ,  2						).reshape(lowercase_ ,  lowercase_						)

						print(F"""Fetching all parameters from the checkpoint at {input_base_path}."""						)
						# Load weights
						if model_size == "7B":
												# Not sharded
												# (The sharded implementation would also work, but this is simpler.)
												lowercase        =							torch.load(os.path.join(lowercase_ ,  """consolidated.00.pth"""						) ,  map_location="""cpu"""						)
						else:
												# Sharded
												lowercase        =							[
												    torch.load(os.path.join(lowercase_ ,  F"""consolidated.{i:02d}.pth"""						) ,  map_location="""cpu"""						)
												    for i in range(lowercase_						)
												]
						lowercase        =							0
						lowercase        =							{"""weight_map""": {}}
						for layer_i in range(lowercase_						):
												lowercase        =							F"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
												if model_size == "7B":
																		# Unsharded
																		lowercase        =							{
																		    F"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
																		        loaded[F"""layers.{layer_i}.attention.wq.weight"""]						),
																		    F"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
																		        loaded[F"""layers.{layer_i}.attention.wk.weight"""]						),
																		    F"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[F"""layers.{layer_i}.attention.wv.weight"""],
																		    F"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[F"""layers.{layer_i}.attention.wo.weight"""],
																		    F"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w1.weight"""],
																		    F"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w2.weight"""],
																		    F"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w3.weight"""],
																		    F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[F"""layers.{layer_i}.attention_norm.weight"""],
																		    F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[F"""layers.{layer_i}.ffn_norm.weight"""],
																		}
												else:
																		# Sharded
																		# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																		# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																		# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																		lowercase        =							{
																		    F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
																		        F"""layers.{layer_i}.attention_norm.weight"""
																		    ].clone(),
																		    F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
																		        F"""layers.{layer_i}.ffn_norm.weight"""
																		    ].clone(),
																		}
																		lowercase        =							permute(
																		    torch.cat(
																		        [
																		            loaded[i][F"""layers.{layer_i}.attention.wq.weight"""].view(lowercase_ ,  lowercase_ ,  lowercase_						)
																		            for i in range(lowercase_						)
																		        ] ,  dim=0 ,  ).reshape(lowercase_ ,  lowercase_						)						)
																		lowercase        =							permute(
																		    torch.cat(
																		        [
																		            loaded[i][F"""layers.{layer_i}.attention.wk.weight"""].view(
																		                lowercase_ ,  lowercase_ ,  lowercase_						)
																		            for i in range(lowercase_						)
																		        ] ,  dim=0 ,  ).reshape(lowercase_ ,  lowercase_						) ,  lowercase_ ,  lowercase_ ,  lowercase_ ,  )
																		lowercase        =							torch.cat(
																		    [
																		        loaded[i][F"""layers.{layer_i}.attention.wv.weight"""].view(
																		            lowercase_ ,  lowercase_ ,  lowercase_						)
																		        for i in range(lowercase_						)
																		    ] ,  dim=0 ,  ).reshape(lowercase_ ,  lowercase_						)

																		lowercase        =							torch.cat(
																		    [loaded[i][F"""layers.{layer_i}.attention.wo.weight"""] for i in range(lowercase_						)] ,  dim=1						)
																		lowercase        =							torch.cat(
																		    [loaded[i][F"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(lowercase_						)] ,  dim=0						)
																		lowercase        =							torch.cat(
																		    [loaded[i][F"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(lowercase_						)] ,  dim=1						)
																		lowercase        =							torch.cat(
																		    [loaded[i][F"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(lowercase_						)] ,  dim=0						)

												lowercase        =							inv_freq
												for k, v in state_dict.items():
																		lowercase        =							filename
																		param_count += v.numel()
												torch.save(lowercase_ ,  os.path.join(lowercase_ ,  lowercase_						)						)

						lowercase        =							F"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
						if model_size == "7B":
												# Unsharded
												lowercase        =							{
												    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
												    """model.norm.weight""": loaded["""norm.weight"""],
												    """lm_head.weight""": loaded["""output.weight"""],
												}
						else:
												lowercase        =							{
												    """model.norm.weight""": loaded[0]["""norm.weight"""],
												    """model.embed_tokens.weight""": torch.cat(
												        [loaded[i]["""tok_embeddings.weight"""] for i in range(lowercase_						)] ,  dim=1						),
												    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(lowercase_						)] ,  dim=0						),
												}

						for k, v in state_dict.items():
												lowercase        =							filename
												param_count += v.numel()
						torch.save(lowercase_ ,  os.path.join(lowercase_ ,  lowercase_						)						)

						# Write configs
						lowercase        =							{"""total_size""": param_count * 2}
						write_json(lowercase_ ,  os.path.join(lowercase_ ,  """pytorch_model.bin.index.json"""						)						)
						lowercase        =							params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
						lowercase        =							params["""multiple_of"""] if """multiple_of""" in params else 256
						lowercase        =							LlamaConfig(
						    hidden_size=lowercase_ ,  intermediate_size=compute_intermediate_size(lowercase_ ,  lowercase_ ,  lowercase_						) ,  num_attention_heads=params["""n_heads"""] ,  num_hidden_layers=params["""n_layers"""] ,  rms_norm_eps=params["""norm_eps"""] ,  num_key_value_heads=lowercase_ ,  )
						config.save_pretrained(lowercase_						)

						# Make space so we can load the model properly now.
						del state_dict
						del loaded
						gc.collect()

						print("""Loading the checkpoint in a Llama model."""						)
						lowercase        =							LlamaForCausalLM.from_pretrained(lowercase_ ,  torch_dtype=torch.floataa ,  low_cpu_mem_usage=lowercase_						)
						# Avoid saving this as part of the config.
						del model.config._name_or_path

						print("""Saving in the Transformers format."""						)
						model.save_pretrained(lowercase_ ,  safe_serialization=lowercase_						)
						shutil.rmtree(lowercase_						)







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      Any ,  lowercase_    :      str						):
						# Initialize the tokenizer based on the `spm` model
						lowercase        =							LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
						print(F"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""						)
						lowercase        =							tokenizer_class(lowercase_						)
						tokenizer.save_pretrained(lowercase_						)







def      SCREAMING_SNAKE_CASE				(				):
						lowercase        =							argparse.ArgumentParser()
						parser.add_argument(
						    """--input_dir""" ,  help="""Location of LLaMA weights, which contains tokenizer.model and model folders""" ,  )
						parser.add_argument(
						    """--model_size""" ,  choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""] ,  )
						parser.add_argument(
						    """--output_dir""" ,  help="""Location to write HF model and tokenizer""" ,  )
						parser.add_argument("""--safe_serialization""" ,  type=lowercase_ ,  help="""Whether or not to save using `safetensors`."""						)
						lowercase        =							parser.parse_args()
						if args.model_size != "tokenizer_only":
												write_model(
												    model_path=args.output_dir ,  input_base_path=os.path.join(args.input_dir ,  args.model_size						) ,  model_size=args.model_size ,  safe_serialization=args.safe_serialization ,  )
						lowercase        =							os.path.join(args.input_dir ,  """tokenizer.model"""						)
						write_tokenizer(args.output_dir ,  lowercase_						)


if __name__ == "__main__":
						main()







'''simple docstring'''


import requests







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      str ,  lowercase_    :      str						):
						lowercase        =							{"""Content-Type""": """application/json"""}
						lowercase        =							requests.post(lowercase_ ,  json={"""text""": message_body} ,  headers=lowercase_						)
						if response.status_code != 200:
												lowercase        =							(
												    """Request to slack returned an error """
												    F"""{response.status_code}, the response is:\n{response.text}"""
												)
												raise ValueError(lowercase_						)


if __name__ == "__main__":
						# Set the slack url to the one provided by Slack when you create the webhook at
						# https://my.slack.com/services/new/incoming-webhook/
						send_slack_message('''<YOUR MESSAGE BODY>''', '''<SLACK CHANNEL URL>''')


