





'''simple docstring'''







import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
					from transformers import LlamaTokenizerFast
except ImportError as e:
					warnings.warn(e)
					warnings.warn(
					    "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
					)
					snake_case_     :       List[Any]						=  None
snake_case_     :       Union[str, Any]						=  {
    "7B": 1_10_08,
    "13B": 1_38_24,
    "30B": 1_79_20,
    "65B": 2_20_16,
    "70B": 2_86_72,
}
snake_case_     :       Dict						=  {
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			List[str],			SCREAMING_SNAKE_CASE__      :			Tuple=1,			SCREAMING_SNAKE_CASE__      :			List[str]=256   ) ->	Dict:
						return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Any   ) ->	Union[str, Any]:
						with open(SCREAMING_SNAKE_CASE__,			'''r'''   ) as f:
												return json.load(SCREAMING_SNAKE_CASE__   )





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			int,			SCREAMING_SNAKE_CASE__      :			Tuple   ) ->	List[Any]:
						with open(SCREAMING_SNAKE_CASE__,			'''w'''   ) as f:
												json.dump(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Dict,			SCREAMING_SNAKE_CASE__      :			Optional[Any],			SCREAMING_SNAKE_CASE__      :			int,			SCREAMING_SNAKE_CASE__      :			List[Any]=True   ) ->	Dict:
						os.makedirs(SCREAMING_SNAKE_CASE__,			exist_ok=SCREAMING_SNAKE_CASE__   )
						UpperCAmelCase_			:      int     						=  os.path.join(SCREAMING_SNAKE_CASE__,			'''tmp'''   )
						os.makedirs(SCREAMING_SNAKE_CASE__,			exist_ok=SCREAMING_SNAKE_CASE__   )

						UpperCAmelCase_			:      Any     						=  read_json(os.path.join(SCREAMING_SNAKE_CASE__,			'''params.json'''   )   )
						UpperCAmelCase_			:      Dict     						=  NUM_SHARDS[model_size]
						UpperCAmelCase_			:      Any     						=  params['''n_layers''']
						UpperCAmelCase_			:      Union[str, Any]     						=  params['''n_heads''']
						UpperCAmelCase_			:      Optional[int]     						=  n_heads // num_shards
						UpperCAmelCase_			:      List[str]     						=  params['''dim''']
						UpperCAmelCase_			:      Tuple     						=  dim // n_heads
						UpperCAmelCase_			:      Optional[Any]     						=  10000.0
						UpperCAmelCase_			:      Optional[int]     						=  1.0 / (base ** (torch.arange(0,			SCREAMING_SNAKE_CASE__,			2   ).float() / dims_per_head))

						if "n_kv_heads" in params:
												UpperCAmelCase_			:      Tuple     						=  params['''n_kv_heads''']  # for GQA / MQA
												UpperCAmelCase_			:      str     						=  n_heads_per_shard // num_key_value_heads
												UpperCAmelCase_			:      str     						=  dim // num_key_value_heads
						else:  # compatibility with other checkpoints
												UpperCAmelCase_			:      List[Any]     						=  n_heads
												UpperCAmelCase_			:      List[Any]     						=  n_heads_per_shard
												UpperCAmelCase_			:      Tuple     						=  dim

						# permute for sliced rotary
						def permute(SCREAMING_SNAKE_CASE__      :			str,			SCREAMING_SNAKE_CASE__      :			int=n_heads,			SCREAMING_SNAKE_CASE__      :			List[Any]=dim,			SCREAMING_SNAKE_CASE__      :			Dict=dim   ):
												return w.view(SCREAMING_SNAKE_CASE__,			dima // n_heads // 2,			2,			SCREAMING_SNAKE_CASE__   ).transpose(1,			2   ).reshape(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )

						print(F"""Fetching all parameters from the checkpoint at {input_base_path}."""   )
						# Load weights
						if model_size == "7B":
												# Not sharded
												# (The sharded implementation would also work, but this is simpler.)
												UpperCAmelCase_			:      int     						=  torch.load(os.path.join(SCREAMING_SNAKE_CASE__,			'''consolidated.00.pth'''   ),			map_location='''cpu'''   )
						else:
												# Sharded
												UpperCAmelCase_			:      List[str]     						=  [
												    torch.load(os.path.join(SCREAMING_SNAKE_CASE__,			F"""consolidated.{i:02d}.pth"""   ),			map_location='''cpu'''   )
												    for i in range(SCREAMING_SNAKE_CASE__   )
												]
						UpperCAmelCase_			:      Dict     						=  0
						UpperCAmelCase_			:      str     						=  {'''weight_map''': {}}
						for layer_i in range(SCREAMING_SNAKE_CASE__   ):
												UpperCAmelCase_			:      Any     						=  F"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
												if model_size == "7B":
																		# Unsharded
																		UpperCAmelCase_			:      Optional[int]     						=  {
																		    F"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
																		        loaded[F"""layers.{layer_i}.attention.wq.weight"""]   ),
																		    F"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
																		        loaded[F"""layers.{layer_i}.attention.wk.weight"""]   ),
																		    F"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[F"""layers.{layer_i}.attention.wv.weight"""],
																		    F"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[F"""layers.{layer_i}.attention.wo.weight"""],
																		    F"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w1.weight"""],
																		    F"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w2.weight"""],
																		    F"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w3.weight"""],
																		    F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[F"""layers.{layer_i}.attention_norm.weight"""],
																		    F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[F"""layers.{layer_i}.ffn_norm.weight"""],
																		}
												else:
																		# Sharded
																		# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																		# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																		# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																		UpperCAmelCase_			:      Optional[int]     						=  {
																		    F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
																		        F"""layers.{layer_i}.attention_norm.weight"""
																		    ].clone(),
																		    F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
																		        F"""layers.{layer_i}.ffn_norm.weight"""
																		    ].clone(),
																		}
																		UpperCAmelCase_			:      Optional[int]     						=  permute(
																		    torch.cat(
																		        [
																		            loaded[i][F"""layers.{layer_i}.attention.wq.weight"""].view(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )
																		            for i in range(SCREAMING_SNAKE_CASE__   )
																		        ],			dim=0,			).reshape(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )   )
																		UpperCAmelCase_			:      Any     						=  permute(
																		    torch.cat(
																		        [
																		            loaded[i][F"""layers.{layer_i}.attention.wk.weight"""].view(
																		                SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )
																		            for i in range(SCREAMING_SNAKE_CASE__   )
																		        ],			dim=0,			).reshape(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   ),			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			)
																		UpperCAmelCase_			:      Any     						=  torch.cat(
																		    [
																		        loaded[i][F"""layers.{layer_i}.attention.wv.weight"""].view(
																		            SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )
																		        for i in range(SCREAMING_SNAKE_CASE__   )
																		    ],			dim=0,			).reshape(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )

																		UpperCAmelCase_			:      Optional[Any]     						=  torch.cat(
																		    [loaded[i][F"""layers.{layer_i}.attention.wo.weight"""] for i in range(SCREAMING_SNAKE_CASE__   )],			dim=1   )
																		UpperCAmelCase_			:      str     						=  torch.cat(
																		    [loaded[i][F"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(SCREAMING_SNAKE_CASE__   )],			dim=0   )
																		UpperCAmelCase_			:      Tuple     						=  torch.cat(
																		    [loaded[i][F"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(SCREAMING_SNAKE_CASE__   )],			dim=1   )
																		UpperCAmelCase_			:      Tuple     						=  torch.cat(
																		    [loaded[i][F"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(SCREAMING_SNAKE_CASE__   )],			dim=0   )

												UpperCAmelCase_			:      Optional[int]     						=  inv_freq
												for k, v in state_dict.items():
																		UpperCAmelCase_			:      Dict     						=  filename
																		param_count += v.numel()
												torch.save(SCREAMING_SNAKE_CASE__,			os.path.join(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )   )

						UpperCAmelCase_			:      str     						=  F"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
						if model_size == "7B":
												# Unsharded
												UpperCAmelCase_			:      Optional[Any]     						=  {
												    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
												    '''model.norm.weight''': loaded['''norm.weight'''],
												    '''lm_head.weight''': loaded['''output.weight'''],
												}
						else:
												UpperCAmelCase_			:      str     						=  {
												    '''model.norm.weight''': loaded[0]['''norm.weight'''],
												    '''model.embed_tokens.weight''': torch.cat(
												        [loaded[i]['''tok_embeddings.weight'''] for i in range(SCREAMING_SNAKE_CASE__   )],			dim=1   ),
												    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(SCREAMING_SNAKE_CASE__   )],			dim=0   ),
												}

						for k, v in state_dict.items():
												UpperCAmelCase_			:      List[str]     						=  filename
												param_count += v.numel()
						torch.save(SCREAMING_SNAKE_CASE__,			os.path.join(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   )   )

						# Write configs
						UpperCAmelCase_			:      Any     						=  {'''total_size''': param_count * 2}
						write_json(SCREAMING_SNAKE_CASE__,			os.path.join(SCREAMING_SNAKE_CASE__,			'''pytorch_model.bin.index.json'''   )   )
						UpperCAmelCase_			:      str     						=  params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
						UpperCAmelCase_			:      List[Any]     						=  params['''multiple_of'''] if '''multiple_of''' in params else 256
						UpperCAmelCase_			:      str     						=  LlamaConfig(
						    hidden_size=SCREAMING_SNAKE_CASE__,			intermediate_size=compute_intermediate_size(SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__,			SCREAMING_SNAKE_CASE__   ),			num_attention_heads=params['''n_heads'''],			num_hidden_layers=params['''n_layers'''],			rms_norm_eps=params['''norm_eps'''],			num_key_value_heads=SCREAMING_SNAKE_CASE__,			)
						config.save_pretrained(SCREAMING_SNAKE_CASE__   )

						# Make space so we can load the model properly now.
						del state_dict
						del loaded
						gc.collect()

						print('''Loading the checkpoint in a Llama model.'''   )
						UpperCAmelCase_			:      int     						=  LlamaForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE__,			torch_dtype=torch.floataa,			low_cpu_mem_usage=SCREAMING_SNAKE_CASE__   )
						# Avoid saving this as part of the config.
						del model.config._name_or_path

						print('''Saving in the Transformers format.'''   )
						model.save_pretrained(SCREAMING_SNAKE_CASE__,			safe_serialization=SCREAMING_SNAKE_CASE__   )
						shutil.rmtree(SCREAMING_SNAKE_CASE__   )





def        lowerCamelCase_      (  SCREAMING_SNAKE_CASE__      :			Dict,			SCREAMING_SNAKE_CASE__      :			Any   ) ->	Optional[Any]:
						# Initialize the tokenizer based on the `spm` model
						UpperCAmelCase_			:      Optional[int]     						=  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
						print(F"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""   )
						UpperCAmelCase_			:      List[str]     						=  tokenizer_class(SCREAMING_SNAKE_CASE__   )
						tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__   )





def        lowerCamelCase_      (  ) ->	str:
						UpperCAmelCase_			:      Optional[Any]     						=  argparse.ArgumentParser()
						parser.add_argument(
						    '''--input_dir''',			help='''Location of LLaMA weights, which contains tokenizer.model and model folders''',			)
						parser.add_argument(
						    '''--model_size''',			choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only'''],			)
						parser.add_argument(
						    '''--output_dir''',			help='''Location to write HF model and tokenizer''',			)
						parser.add_argument('''--safe_serialization''',			type=SCREAMING_SNAKE_CASE__,			help='''Whether or not to save using `safetensors`.'''   )
						UpperCAmelCase_			:      Union[str, Any]     						=  parser.parse_args()
						if args.model_size != "tokenizer_only":
												write_model(
												    model_path=args.output_dir,			input_base_path=os.path.join(args.input_dir,			args.model_size   ),			model_size=args.model_size,			safe_serialization=args.safe_serialization,			)
						UpperCAmelCase_			:      Any     						=  os.path.join(args.input_dir,			'''tokenizer.model'''   )
						write_tokenizer(args.output_dir,			SCREAMING_SNAKE_CASE__   )


if __name__ == "__main__":
					main()












'''simple docstring'''







import unittest

import numpy as np
import torch

from diffusers import KarrasVePipeline, KarrasVeScheduler, UNetaDModel
from diffusers.utils.testing_utils import enable_full_determinism, require_torch, slow, torch_device


enable_full_determinism()


class 					__a      (unittest.TestCase     ):

							@property
							def     UpperCAmelCase__	(							self							:     Dict   )					->       str:







													"""simple docstring"""
													torch.manual_seed(0   )
													UpperCAmelCase_			:      Optional[Any]     						=  UNetaDModel(
													    block_out_channels=(32, 64)       ,     layers_per_block=2       ,     sample_size=32       ,     in_channels=3       ,     out_channels=3       ,     down_block_types=('''DownBlock2D''', '''AttnDownBlock2D''')       ,     up_block_types=('''AttnUpBlock2D''', '''UpBlock2D''')       ,     )
													return model



							def     UpperCAmelCase__	(							self							:     Dict   )					->       Tuple:







													"""simple docstring"""
													UpperCAmelCase_			:      List[Any]     						=  self.dummy_uncond_unet
													UpperCAmelCase_			:      Dict     						=  KarrasVeScheduler()

													UpperCAmelCase_			:      Union[str, Any]     						=  KarrasVePipeline(unet=__magic_name__       ,     scheduler=__magic_name__   )
													pipe.to(__magic_name__   )
													pipe.set_progress_bar_config(disable=__magic_name__   )

													UpperCAmelCase_			:      Dict     						=  torch.manual_seed(0   )
													UpperCAmelCase_			:      Optional[int]     						=  pipe(num_inference_steps=2       ,     generator=__magic_name__       ,     output_type='''numpy'''   ).images

													UpperCAmelCase_			:      Tuple     						=  torch.manual_seed(0   )
													UpperCAmelCase_			:      str     						=  pipe(num_inference_steps=2       ,     generator=__magic_name__       ,     output_type='''numpy'''       ,     return_dict=__magic_name__   )[0]

													UpperCAmelCase_			:      Union[str, Any]     						=  image[0, -3:, -3:, -1]
													UpperCAmelCase_			:      Optional[Any]     						=  image_from_tuple[0, -3:, -3:, -1]

													assert image.shape == (1, 32, 32, 3)
													UpperCAmelCase_			:      Dict     						=  np.array([0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0]   )

													assert np.abs(image_slice.flatten() - expected_slice   ).max() < 1E-2
													assert np.abs(image_from_tuple_slice.flatten() - expected_slice   ).max() < 1E-2





@slow
@require_torch
class 					__a      (unittest.TestCase     ):

							def     UpperCAmelCase__	(							self							:     int   )					->       Union[str, Any]:







													"""simple docstring"""
													UpperCAmelCase_			:      List[str]     						=  '''google/ncsnpp-celebahq-256'''
													UpperCAmelCase_			:      List[str]     						=  UNetaDModel.from_pretrained(__magic_name__   )
													UpperCAmelCase_			:      List[Any]     						=  KarrasVeScheduler()

													UpperCAmelCase_			:      Any     						=  KarrasVePipeline(unet=__magic_name__       ,     scheduler=__magic_name__   )
													pipe.to(__magic_name__   )
													pipe.set_progress_bar_config(disable=__magic_name__   )

													UpperCAmelCase_			:      Dict     						=  torch.manual_seed(0   )
													UpperCAmelCase_			:      Optional[Any]     						=  pipe(num_inference_steps=20       ,     generator=__magic_name__       ,     output_type='''numpy'''   ).images

													UpperCAmelCase_			:      Any     						=  image[0, -3:, -3:, -1]
													assert image.shape == (1, 2_56, 2_56, 3)
													UpperCAmelCase_			:      Optional[Any]     						=  np.array([0.5_7_8, 0.5_8_1_1, 0.5_9_2_4, 0.5_8_0_9, 0.5_8_7, 0.5_8_8_6, 0.5_8_6_1, 0.5_8_0_2, 0.5_8_6]   )

													assert np.abs(image_slice.flatten() - expected_slice   ).max() < 1E-2





