

import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
       from transformers import LlamaTokenizerFast
except ImportError as e:
       warnings.warn(e)
       warnings.warn(
           "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
       )
       UpperCamelCase__  :      Optional[Any]			=    None
UpperCamelCase__  :      str			=    {
    "7B": 11_008,
    "13B": 13_824,
    "30B": 17_920,
    "65B": 22_016,
    "70B": 28_672,
}
UpperCamelCase__  :      str			=    {
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Union[str, Any] ,   _SCREAMING_SNAKE_CASE					:		Optional[Any]=1 ,   _SCREAMING_SNAKE_CASE					:		Union[str, Any]=256       ):





    """simple docstring"""


    return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3       )       ) + multiple_of - 1) // multiple_of)


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Optional[int]       ):





    """simple docstring"""


    with open(_SCREAMING_SNAKE_CASE ,   'r'       ) as f:
        return json.load(_SCREAMING_SNAKE_CASE       )


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Optional[int] ,   _SCREAMING_SNAKE_CASE					:		str       ):





    """simple docstring"""


    with open(_SCREAMING_SNAKE_CASE ,   'w'       ) as f:
        json.dump(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Tuple ,   _SCREAMING_SNAKE_CASE					:		Tuple ,   _SCREAMING_SNAKE_CASE					:		Any ,   _SCREAMING_SNAKE_CASE					:		Optional[int]=True       ):





    """simple docstring"""


    os.makedirs(_SCREAMING_SNAKE_CASE ,   exist_ok=_SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_									=						os.path.join(_SCREAMING_SNAKE_CASE ,   'tmp'       )
    os.makedirs(_SCREAMING_SNAKE_CASE ,   exist_ok=_SCREAMING_SNAKE_CASE       )

    SCREAMING_SNAKE_CASE_									=						read_json(os.path.join(_SCREAMING_SNAKE_CASE ,   'params.json'       )       )
    SCREAMING_SNAKE_CASE_									=						NUM_SHARDS[model_size]
    SCREAMING_SNAKE_CASE_									=						params['n_layers']
    SCREAMING_SNAKE_CASE_									=						params['n_heads']
    SCREAMING_SNAKE_CASE_									=						n_heads // num_shards
    SCREAMING_SNAKE_CASE_									=						params['dim']
    SCREAMING_SNAKE_CASE_									=						dim // n_heads
    SCREAMING_SNAKE_CASE_									=						10000.0
    SCREAMING_SNAKE_CASE_									=						1.0 / (base ** (torch.arange(0 ,   _SCREAMING_SNAKE_CASE ,   2       ).float() / dims_per_head))

    if "n_kv_heads" in params:
        SCREAMING_SNAKE_CASE_									=						params['n_kv_heads']  # for GQA / MQA
        SCREAMING_SNAKE_CASE_									=						n_heads_per_shard // num_key_value_heads
        SCREAMING_SNAKE_CASE_									=						dim // num_key_value_heads
    else:  # compatibility with other checkpoints
        SCREAMING_SNAKE_CASE_									=						n_heads
        SCREAMING_SNAKE_CASE_									=						n_heads_per_shard
        SCREAMING_SNAKE_CASE_									=						dim

    # permute for sliced rotary
    def permute(_SCREAMING_SNAKE_CASE					:		List[Any] ,   _SCREAMING_SNAKE_CASE					:		List[str]=n_heads ,   _SCREAMING_SNAKE_CASE					:		Dict=dim ,   _SCREAMING_SNAKE_CASE					:		List[Any]=dim       ):
        return w.view(_SCREAMING_SNAKE_CASE ,   dima // n_heads // 2 ,   2 ,   _SCREAMING_SNAKE_CASE       ).transpose(1 ,   2       ).reshape(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )

    print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""       )
    # Load weights
    if model_size == "7B":
        # Not sharded
        # (The sharded implementation would also work, but this is simpler.)
        SCREAMING_SNAKE_CASE_									=						torch.load(os.path.join(_SCREAMING_SNAKE_CASE ,   'consolidated.00.pth'       ) ,   map_location='cpu'       )
    else:
        # Sharded
        SCREAMING_SNAKE_CASE_									=						[
            torch.load(os.path.join(_SCREAMING_SNAKE_CASE ,   f"""consolidated.{i:02d}.pth"""       ) ,   map_location='cpu'       )
            for i in range(_SCREAMING_SNAKE_CASE       )
        ]
    SCREAMING_SNAKE_CASE_									=						0
    SCREAMING_SNAKE_CASE_									=						{'weight_map': {}}
    for layer_i in range(_SCREAMING_SNAKE_CASE       ):
        SCREAMING_SNAKE_CASE_									=						f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
        if model_size == "7B":
            # Unsharded
            SCREAMING_SNAKE_CASE_									=						{
                f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
                    loaded[f"""layers.{layer_i}.attention.wq.weight"""]       ),
                f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
                    loaded[f"""layers.{layer_i}.attention.wk.weight"""]       ),
                f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
                f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
                f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
                f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
                f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
                f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
                f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
            }
        else:
            # Sharded
            # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
            # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
            # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

            SCREAMING_SNAKE_CASE_									=						{
                f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
                    f"""layers.{layer_i}.attention_norm.weight"""
                ].clone(),
                f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
                    f"""layers.{layer_i}.ffn_norm.weight"""
                ].clone(),
            }
            SCREAMING_SNAKE_CASE_									=						permute(
                torch.cat(
                    [
                        loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
                        for i in range(_SCREAMING_SNAKE_CASE       )
                    ] ,   dim=0 ,   ).reshape(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )       )
            SCREAMING_SNAKE_CASE_									=						permute(
                torch.cat(
                    [
                        loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
                            _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
                        for i in range(_SCREAMING_SNAKE_CASE       )
                    ] ,   dim=0 ,   ).reshape(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       ) ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   )
            SCREAMING_SNAKE_CASE_									=						torch.cat(
                [
                    loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
                        _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
                    for i in range(_SCREAMING_SNAKE_CASE       )
                ] ,   dim=0 ,   ).reshape(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )

            SCREAMING_SNAKE_CASE_									=						torch.cat(
                [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(_SCREAMING_SNAKE_CASE       )] ,   dim=1       )
            SCREAMING_SNAKE_CASE_									=						torch.cat(
                [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(_SCREAMING_SNAKE_CASE       )] ,   dim=0       )
            SCREAMING_SNAKE_CASE_									=						torch.cat(
                [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(_SCREAMING_SNAKE_CASE       )] ,   dim=1       )
            SCREAMING_SNAKE_CASE_									=						torch.cat(
                [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(_SCREAMING_SNAKE_CASE       )] ,   dim=0       )

        SCREAMING_SNAKE_CASE_									=						inv_freq
        for k, v in state_dict.items():
            SCREAMING_SNAKE_CASE_									=						filename
            param_count += v.numel()
        torch.save(_SCREAMING_SNAKE_CASE ,   os.path.join(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )       )

    SCREAMING_SNAKE_CASE_									=						f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
    if model_size == "7B":
        # Unsharded
        SCREAMING_SNAKE_CASE_									=						{
            'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
            'model.norm.weight': loaded['norm.weight'],
            'lm_head.weight': loaded['output.weight'],
        }
    else:
        SCREAMING_SNAKE_CASE_									=						{
            'model.norm.weight': loaded[0]['norm.weight'],
            'model.embed_tokens.weight': torch.cat(
                [loaded[i]['tok_embeddings.weight'] for i in range(_SCREAMING_SNAKE_CASE       )] ,   dim=1       ),
            'lm_head.weight': torch.cat([loaded[i]['output.weight'] for i in range(_SCREAMING_SNAKE_CASE       )] ,   dim=0       ),
        }

    for k, v in state_dict.items():
        SCREAMING_SNAKE_CASE_									=						filename
        param_count += v.numel()
    torch.save(_SCREAMING_SNAKE_CASE ,   os.path.join(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )       )

    # Write configs
    SCREAMING_SNAKE_CASE_									=						{'total_size': param_count * 2}
    write_json(_SCREAMING_SNAKE_CASE ,   os.path.join(_SCREAMING_SNAKE_CASE ,   'pytorch_model.bin.index.json'       )       )
    SCREAMING_SNAKE_CASE_									=						params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
    SCREAMING_SNAKE_CASE_									=						params['multiple_of'] if 'multiple_of' in params else 256
    SCREAMING_SNAKE_CASE_									=						LlamaConfig(
        hidden_size=_SCREAMING_SNAKE_CASE ,   intermediate_size=compute_intermediate_size(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       ) ,   num_attention_heads=params['n_heads'] ,   num_hidden_layers=params['n_layers'] ,   rms_norm_eps=params['norm_eps'] ,   num_key_value_heads=_SCREAMING_SNAKE_CASE ,   )
    config.save_pretrained(_SCREAMING_SNAKE_CASE       )

    # Make space so we can load the model properly now.
    del state_dict
    del loaded
    gc.collect()

    print('Loading the checkpoint in a Llama model.'       )
    SCREAMING_SNAKE_CASE_									=						LlamaForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE ,   torch_dtype=torch.floataa ,   low_cpu_mem_usage=_SCREAMING_SNAKE_CASE       )
    # Avoid saving this as part of the config.
    del model.config._name_or_path

    print('Saving in the Transformers format.'       )
    model.save_pretrained(_SCREAMING_SNAKE_CASE ,   safe_serialization=_SCREAMING_SNAKE_CASE       )
    shutil.rmtree(_SCREAMING_SNAKE_CASE       )


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Union[str, Any] ,   _SCREAMING_SNAKE_CASE					:		Tuple       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
    print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""       )
    SCREAMING_SNAKE_CASE_									=						tokenizer_class(_SCREAMING_SNAKE_CASE       )
    tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE       )


def 					_UpperCAmelCase       (  ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						argparse.ArgumentParser()
    parser.add_argument(
        '--input_dir' ,   help='Location of LLaMA weights, which contains tokenizer.model and model folders' ,   )
    parser.add_argument(
        '--model_size' ,   choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only'] ,   )
    parser.add_argument(
        '--output_dir' ,   help='Location to write HF model and tokenizer' ,   )
    parser.add_argument('--safe_serialization' ,   type=_SCREAMING_SNAKE_CASE ,   help='Whether or not to save using `safetensors`.'       )
    SCREAMING_SNAKE_CASE_									=						parser.parse_args()
    if args.model_size != "tokenizer_only":
        write_model(
            model_path=args.output_dir ,   input_base_path=os.path.join(args.input_dir ,   args.model_size       ) ,   model_size=args.model_size ,   safe_serialization=args.safe_serialization ,   )
    SCREAMING_SNAKE_CASE_									=						os.path.join(args.input_dir ,   'tokenizer.model'       )
    write_tokenizer(args.output_dir ,   _SCREAMING_SNAKE_CASE       )


if __name__ == "__main__":
       main()





def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		int = 200       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						[1, 2, 5, 10, 20, 50, 100, 200]
    SCREAMING_SNAKE_CASE_									=						[0] * (pence + 1)
    SCREAMING_SNAKE_CASE_									=						1  # base case: 1 way to make 0 pence

    for coin in coins:
        for i in range(_SCREAMING_SNAKE_CASE ,   pence + 1 ,   1       ):
            number_of_ways[i] += number_of_ways[i - coin]
    return number_of_ways[pence]


if __name__ == "__main__":
       assert solution(200) == 73_682


