"""simple docstring"""




import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
     from transformers import LlamaTokenizerFast
except ImportError as e:
     warnings.warn(e)
     warnings.warn(
         'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
     )
     _a   :	Optional[int] 					=     None
_a   :	Tuple 					=     {
    '7B': 11_008,
    '13B': 13_824,
    '30B': 17_920,
    '65B': 22_016,
    '70B': 28_672,
}
_a   :	List[str] 					=     {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}





def 							SCREAMING_SNAKE_CASE  (    _lowerCamelCase		: Optional[Any]    ,_lowerCamelCase		: int=1    ,_lowerCamelCase		: List[Any]=256			) ->   Any:
  return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3			)			) + multiple_of - 1) // multiple_of)





def 							SCREAMING_SNAKE_CASE  (    _lowerCamelCase		: List[str]			) ->   List[str]:
  with open(_lowerCamelCase    ,"""r"""			) as f:
    return json.load(_lowerCamelCase			)





def 							SCREAMING_SNAKE_CASE  (    _lowerCamelCase		: Union[str, Any]    ,_lowerCamelCase		: Any			) ->   str:
  with open(_lowerCamelCase    ,"""w"""			) as f:
    json.dump(_lowerCamelCase    ,_lowerCamelCase			)





def 							SCREAMING_SNAKE_CASE  (    _lowerCamelCase		: List[Any]    ,_lowerCamelCase		: Tuple    ,_lowerCamelCase		: List[str]    ,_lowerCamelCase		: List[Any]=True			) ->   Optional[Any]:
  os.makedirs(_lowerCamelCase    ,exist_ok=_lowerCamelCase			)
  _lowerCAmelCase		: List[Any]						    =  os.path.join(_lowerCamelCase    ,"""tmp"""			)
  os.makedirs(_lowerCamelCase    ,exist_ok=_lowerCamelCase			)

  _lowerCAmelCase		: Any						    =  read_json(os.path.join(_lowerCamelCase    ,"""params.json"""			)			)
  _lowerCAmelCase		: List[Any]						    =  NUM_SHARDS[model_size]
  _lowerCAmelCase		: List[str]						    =  params["""n_layers"""]
  _lowerCAmelCase		: Tuple						    =  params["""n_heads"""]
  _lowerCAmelCase		: Tuple						    =  n_heads // num_shards
  _lowerCAmelCase		: List[Any]						    =  params["""dim"""]
  _lowerCAmelCase		: Optional[Any]						    =  dim // n_heads
  _lowerCAmelCase		: int						    =  1_00_00.0
  _lowerCAmelCase		: Any						    =  1.0 / (base ** (torch.arange(0    ,_lowerCamelCase    ,2			).float() / dims_per_head))

  if "n_kv_heads" in params:
    _lowerCAmelCase		: Optional[Any]						    =  params["""n_kv_heads"""]  # for GQA / MQA
    _lowerCAmelCase		: Optional[int]						    =  n_heads_per_shard // num_key_value_heads
    _lowerCAmelCase		: Any						    =  dim // num_key_value_heads
  else:  # compatibility with other checkpoints
    _lowerCAmelCase		: int						    =  n_heads
    _lowerCAmelCase		: Optional[Any]						    =  n_heads_per_shard
    _lowerCAmelCase		: List[Any]						    =  dim

  # permute for sliced rotary
  def permute(_lowerCamelCase		: Optional[Any]    ,_lowerCamelCase		: Union[str, Any]=n_heads    ,_lowerCamelCase		: Union[str, Any]=dim    ,_lowerCamelCase		: Tuple=dim			):
    return w.view(_lowerCamelCase    ,dima // n_heads // 2    ,2    ,_lowerCamelCase			).transpose(1    ,2			).reshape(_lowerCamelCase    ,_lowerCamelCase			)

  print(f"Fetching all parameters from the checkpoint at {input_base_path}."			)
  # Load weights
  if model_size == "7B":
    # Not sharded
    # (The sharded implementation would also work, but this is simpler.)
    _lowerCAmelCase		: Union[str, Any]						    =  torch.load(os.path.join(_lowerCamelCase    ,"""consolidated.00.pth"""			)    ,map_location="""cpu"""			)
  else:
    # Sharded
    _lowerCAmelCase		: List[Any]						    =  [
        torch.load(os.path.join(_lowerCamelCase    ,f"consolidated.{i:02d}.pth"			)    ,map_location="""cpu"""			)
        for i in range(_lowerCamelCase			)
    ]
  _lowerCAmelCase		: Optional[int]						    =  0
  _lowerCAmelCase		: Tuple						    =  {"""weight_map""": {}}
  for layer_i in range(_lowerCamelCase			):
    _lowerCAmelCase		: Union[str, Any]						    =  f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
    if model_size == "7B":
      # Unsharded
      _lowerCAmelCase		: Tuple						    =  {
          f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
              loaded[f"layers.{layer_i}.attention.wq.weight"]			),
          f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
              loaded[f"layers.{layer_i}.attention.wk.weight"]			),
          f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
          f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
          f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
          f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
          f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
          f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
          f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
      }
    else:
      # Sharded
      # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
      # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
      # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

      _lowerCAmelCase		: List[Any]						    =  {
          f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
              f"layers.{layer_i}.attention_norm.weight"
          ].clone(),
          f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
              f"layers.{layer_i}.ffn_norm.weight"
          ].clone(),
      }
      _lowerCAmelCase		: List[str]						    =  permute(
          torch.cat(
              [
                  loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(_lowerCamelCase    ,_lowerCamelCase    ,_lowerCamelCase			)
                  for i in range(_lowerCamelCase			)
              ]    ,dim=0    ,).reshape(_lowerCamelCase    ,_lowerCamelCase			)			)
      _lowerCAmelCase		: int						    =  permute(
          torch.cat(
              [
                  loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(
                      _lowerCamelCase    ,_lowerCamelCase    ,_lowerCamelCase			)
                  for i in range(_lowerCamelCase			)
              ]    ,dim=0    ,).reshape(_lowerCamelCase    ,_lowerCamelCase			)    ,_lowerCamelCase    ,_lowerCamelCase    ,_lowerCamelCase    ,)
      _lowerCAmelCase		: Optional[int]						    =  torch.cat(
          [
              loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(
                  _lowerCamelCase    ,_lowerCamelCase    ,_lowerCamelCase			)
              for i in range(_lowerCamelCase			)
          ]    ,dim=0    ,).reshape(_lowerCamelCase    ,_lowerCamelCase			)

      _lowerCAmelCase		: Dict						    =  torch.cat(
          [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(_lowerCamelCase			)]    ,dim=1			)
      _lowerCAmelCase		: Any						    =  torch.cat(
          [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(_lowerCamelCase			)]    ,dim=0			)
      _lowerCAmelCase		: Optional[int]						    =  torch.cat(
          [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(_lowerCamelCase			)]    ,dim=1			)
      _lowerCAmelCase		: List[Any]						    =  torch.cat(
          [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(_lowerCamelCase			)]    ,dim=0			)

    _lowerCAmelCase		: str						    =  inv_freq
    for k, v in state_dict.items():
      _lowerCAmelCase		: Union[str, Any]						    =  filename
      param_count += v.numel()
    torch.save(_lowerCamelCase    ,os.path.join(_lowerCamelCase    ,_lowerCamelCase			)			)

  _lowerCAmelCase		: Union[str, Any]						    =  f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
  if model_size == "7B":
    # Unsharded
    _lowerCAmelCase		: Dict						    =  {
        """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
        """model.norm.weight""": loaded["""norm.weight"""],
        """lm_head.weight""": loaded["""output.weight"""],
    }
  else:
    _lowerCAmelCase		: int						    =  {
        """model.norm.weight""": loaded[0]["""norm.weight"""],
        """model.embed_tokens.weight""": torch.cat(
            [loaded[i]["""tok_embeddings.weight"""] for i in range(_lowerCamelCase			)]    ,dim=1			),
        """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(_lowerCamelCase			)]    ,dim=0			),
    }

  for k, v in state_dict.items():
    _lowerCAmelCase		: Dict						    =  filename
    param_count += v.numel()
  torch.save(_lowerCamelCase    ,os.path.join(_lowerCamelCase    ,_lowerCamelCase			)			)

  # Write configs
  _lowerCAmelCase		: int						    =  {"""total_size""": param_count * 2}
  write_json(_lowerCamelCase    ,os.path.join(_lowerCamelCase    ,"""pytorch_model.bin.index.json"""			)			)
  _lowerCAmelCase		: Any						    =  params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
  _lowerCAmelCase		: Optional[int]						    =  params["""multiple_of"""] if """multiple_of""" in params else 256
  _lowerCAmelCase		: Optional[Any]						    =  LlamaConfig(
      hidden_size=_lowerCamelCase    ,intermediate_size=compute_intermediate_size(_lowerCamelCase    ,_lowerCamelCase    ,_lowerCamelCase			)    ,num_attention_heads=params["""n_heads"""]    ,num_hidden_layers=params["""n_layers"""]    ,rms_norm_eps=params["""norm_eps"""]    ,num_key_value_heads=_lowerCamelCase    ,)
  config.save_pretrained(_lowerCamelCase			)

  # Make space so we can load the model properly now.
  del state_dict
  del loaded
  gc.collect()

  print("""Loading the checkpoint in a Llama model."""			)
  _lowerCAmelCase		: Optional[Any]						    =  LlamaForCausalLM.from_pretrained(_lowerCamelCase    ,torch_dtype=torch.floataa    ,low_cpu_mem_usage=_lowerCamelCase			)
  # Avoid saving this as part of the config.
  del model.config._name_or_path

  print("""Saving in the Transformers format."""			)
  model.save_pretrained(_lowerCamelCase    ,safe_serialization=_lowerCamelCase			)
  shutil.rmtree(_lowerCamelCase			)





def 							SCREAMING_SNAKE_CASE  (    _lowerCamelCase		: Optional[Any]    ,_lowerCamelCase		: Union[str, Any]			) ->   List[Any]:
  # Initialize the tokenizer based on the `spm` model
  _lowerCAmelCase		: List[Any]						    =  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
  print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}."			)
  _lowerCAmelCase		: Union[str, Any]						    =  tokenizer_class(_lowerCamelCase			)
  tokenizer.save_pretrained(_lowerCamelCase			)





def 							SCREAMING_SNAKE_CASE  (    ) ->   Dict:
  _lowerCAmelCase		: str						    =  argparse.ArgumentParser()
  parser.add_argument(
      """--input_dir"""    ,help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""    ,)
  parser.add_argument(
      """--model_size"""    ,choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]    ,)
  parser.add_argument(
      """--output_dir"""    ,help="""Location to write HF model and tokenizer"""    ,)
  parser.add_argument("""--safe_serialization"""    ,type=_lowerCamelCase    ,help="""Whether or not to save using `safetensors`."""			)
  _lowerCAmelCase		: Union[str, Any]						    =  parser.parse_args()
  if args.model_size != "tokenizer_only":
    write_model(
        model_path=args.output_dir    ,input_base_path=os.path.join(args.input_dir    ,args.model_size			)    ,model_size=args.model_size    ,safe_serialization=args.safe_serialization    ,)
  _lowerCAmelCase		: str						    =  os.path.join(args.input_dir    ,"""tokenizer.model"""			)
  write_tokenizer(args.output_dir    ,_lowerCamelCase			)


if __name__ == "__main__":
     main()





"""simple docstring"""




import unittest

import numpy as np

from transformers import MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING, TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING
from transformers.pipelines import AudioClassificationPipeline, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_torchaudio,
    slow,
)

from .test_pipelines_common import ANY
@is_pipeline_test
class 		__A							(				unittest.TestCase	):
     _UpperCamelCase							:    str	      =	MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING
     _UpperCamelCase							:    Any	      =	TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING






     def 					__A						(	self       ,  a__       ,  a__       ,  a__						):
       _lowerCAmelCase		: List[Any]						    =  AudioClassificationPipeline(model=a__       ,  feature_extractor=a__						)

       # test with a raw waveform
       _lowerCAmelCase		: Optional[int]						    =  np.zeros((34000,)						)
       _lowerCAmelCase		: Optional[Any]						    =  np.zeros((14000,)						)
       return audio_classifier, [audioa, audio]






     def 					__A						(	self       ,  a__       ,  a__						):
       _lowerCAmelCase    ,			_lowerCAmelCase		: List[Any]						    =  examples
       _lowerCAmelCase		: List[Any]						    =  audio_classifier(a__						)
       # by default a model is initialized with num_labels=2
       self.assertEqual(
           a__       ,  [
               {"""score""": ANY(a__						), """label""": ANY(a__						)},
               {"""score""": ANY(a__						), """label""": ANY(a__						)},
           ]       ,  )
       _lowerCAmelCase		: Tuple						    =  audio_classifier(a__       ,  top_k=1						)
       self.assertEqual(
           a__       ,  [
               {"""score""": ANY(a__						), """label""": ANY(a__						)},
           ]       ,  )

       self.run_torchaudio(a__						)






     @require_torchaudio
     def 					__A						(	self       ,  a__						):
       import datasets

       # test with a local file
       _lowerCAmelCase		: int						    =  datasets.load_dataset("""hf-internal-testing/librispeech_asr_dummy"""       ,  """clean"""       ,  split="""validation"""						)
       _lowerCAmelCase		: List[Any]						    =  dataset[0]["""audio"""]["""array"""]
       _lowerCAmelCase		: str						    =  audio_classifier(a__						)
       self.assertEqual(
           a__       ,  [
               {"""score""": ANY(a__						), """label""": ANY(a__						)},
               {"""score""": ANY(a__						), """label""": ANY(a__						)},
           ]       ,  )






     @require_torch
     def 					__A						(	self						):
       _lowerCAmelCase		: int						    =  """anton-l/wav2vec2-random-tiny-classifier"""

       _lowerCAmelCase		: Optional[Any]						    =  pipeline("""audio-classification"""       ,  model=a__						)

       _lowerCAmelCase		: Any						    =  np.ones((8000,)						)
       _lowerCAmelCase		: List[str]						    =  audio_classifier(a__       ,  top_k=4						)

       _lowerCAmelCase		: List[str]						    =  [
           {"""score""": 0.0_8_4_2, """label""": """no"""},
           {"""score""": 0.0_8_3_8, """label""": """up"""},
           {"""score""": 0.0_8_3_7, """label""": """go"""},
           {"""score""": 0.0_8_3_4, """label""": """right"""},
       ]
       _lowerCAmelCase		: str						    =  [
           {"""score""": 0.0_8_4_5, """label""": """stop"""},
           {"""score""": 0.0_8_4_4, """label""": """on"""},
           {"""score""": 0.0_8_4_1, """label""": """right"""},
           {"""score""": 0.0_8_3_4, """label""": """left"""},
       ]
       self.assertIn(nested_simplify(a__       ,  decimals=4						)       ,  [EXPECTED_OUTPUT, EXPECTED_OUTPUT_PT_2]						)

       _lowerCAmelCase		: int						    =  {"""array""": np.ones((8000,)						), """sampling_rate""": audio_classifier.feature_extractor.sampling_rate}
       _lowerCAmelCase		: int						    =  audio_classifier(a__       ,  top_k=4						)
       self.assertIn(nested_simplify(a__       ,  decimals=4						)       ,  [EXPECTED_OUTPUT, EXPECTED_OUTPUT_PT_2]						)






     @require_torch
     @slow
     def 					__A						(	self						):
       import datasets

       _lowerCAmelCase		: Optional[Any]						    =  """superb/wav2vec2-base-superb-ks"""

       _lowerCAmelCase		: List[str]						    =  pipeline("""audio-classification"""       ,  model=a__						)
       _lowerCAmelCase		: str						    =  datasets.load_dataset("""anton-l/superb_dummy"""       ,  """ks"""       ,  split="""test"""						)

       _lowerCAmelCase		: Optional[Any]						    =  np.array(dataset[3]["""speech"""]       ,  dtype=np.floataa						)
       _lowerCAmelCase		: List[str]						    =  audio_classifier(a__       ,  top_k=4						)
       self.assertEqual(
           nested_simplify(a__       ,  decimals=3						)       ,  [
               {"""score""": 0.9_8_1, """label""": """go"""},
               {"""score""": 0.0_0_7, """label""": """up"""},
               {"""score""": 0.0_0_6, """label""": """_unknown_"""},
               {"""score""": 0.0_0_1, """label""": """down"""},
           ]       ,  )






     @require_tf
     @unittest.skip("""Audio classification is not implemented for TF"""						)
     def 					__A						(	self						):
       pass




