
"""simple docstring"""




import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
    import bitsandbytes as bnb
    import torch
    import torch.nn as nn

    from ..pytorch_utils import ConvaD

if is_accelerate_available():
    from accelerate import init_empty_weights
    from accelerate.utils import find_tied_parameters

A__      :	int          =    logging.get_logger(__name__)



def 							a__ (				lowerCAmelCase	:       str       ,    lowerCAmelCase	:       Union[str, Any]       ,    lowerCAmelCase	:       str       ,    lowerCAmelCase	:       List[Any]=None       ,    lowerCAmelCase	:       Optional[int]=None		):






     '''simple docstring'''
     # Recurse if needed
     if "." in tensor_name:
          UpperCAmelCase__							: Dict 							=						tensor_name.split("."		)
          for split in splits[:-1]:
               UpperCAmelCase__							: int 							=						getattr(_lowercase       ,    _lowercase		)
               if new_module is None:
                    raise ValueError(F"{module} has no attribute {split}."		)
               UpperCAmelCase__							: List[str] 							=						new_module
          UpperCAmelCase__							: Dict 							=						splits[-1]

     if tensor_name not in module._parameters and tensor_name not in module._buffers:
          raise ValueError(F"{module} does not have a parameter or a buffer named {tensor_name}."		)
     UpperCAmelCase__							: Union[str, Any] 							=						tensor_name in module._buffers
     UpperCAmelCase__							: Optional[Any] 							=						getattr(_lowercase       ,    _lowercase		)

     if old_value.device == torch.device("meta"		) and device not in ["meta", torch.device("meta"		)] and value is None:
          raise ValueError(F"{tensor_name} is on the meta device, we need a `value` to put in on {device}."		)

     UpperCAmelCase__							: Optional[int] 							=						False
     UpperCAmelCase__							: Dict 							=						False
     if is_buffer or not is_bitsandbytes_available():
          UpperCAmelCase__							: List[str] 							=						False
          UpperCAmelCase__							: List[str] 							=						False
     else:
          UpperCAmelCase__							: Dict 							=						hasattr(bnb.nn       ,    "Params4bit"		) and isinstance(module._parameters[tensor_name]       ,    bnb.nn.Paramsabit		)
          UpperCAmelCase__							: Any 							=						isinstance(module._parameters[tensor_name]       ,    bnb.nn.IntaParams		)

     if is_abit or is_abit:
          UpperCAmelCase__							: List[str] 							=						module._parameters[tensor_name]
          if param.device.type != "cuda":
               if value is None:
                    UpperCAmelCase__							: Any 							=						old_value.to(_lowercase		)
               elif isinstance(_lowercase       ,    torch.Tensor		):
                    UpperCAmelCase__							: List[Any] 							=						value.to("cpu"		)
                    if value.dtype == torch.inta:
                         UpperCAmelCase__							: List[Any] 							=						version.parse(importlib.metadata.version("bitsandbytes"		)		) > version.parse(
                             "0.37.2"		)
                         if not is_abit_serializable:
                              raise ValueError(
                                  "Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. "
                                  "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."		)
               else:
                    UpperCAmelCase__							: Dict 							=						torch.tensor(_lowercase       ,    device="cpu"		)

               # Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
               # Since weights are saved in the correct "orientation", we skip transposing when loading.
               if issubclass(module.source_cls       ,    _lowercase		) and fpaa_statistics is None:
                    UpperCAmelCase__							: Optional[int] 							=						new_value.T

               UpperCAmelCase__							: Dict 							=						old_value.__dict__
               if is_abit:
                    UpperCAmelCase__							: List[Any] 							=						bnb.nn.IntaParams(_lowercase       ,    requires_grad=_lowercase       ,    **_lowercase		).to(_lowercase		)
               elif is_abit:
                    UpperCAmelCase__							: str 							=						bnb.nn.Paramsabit(_lowercase       ,    requires_grad=_lowercase       ,    **_lowercase		).to(_lowercase		)

               UpperCAmelCase__							: Tuple 							=						new_value
               if fpaa_statistics is not None:
                    setattr(module.weight       ,    "SCB"       ,    fpaa_statistics.to(_lowercase		)		)

     else:
          if value is None:
               UpperCAmelCase__							: Union[str, Any] 							=						old_value.to(_lowercase		)
          elif isinstance(_lowercase       ,    torch.Tensor		):
               UpperCAmelCase__							: List[Any] 							=						value.to(_lowercase		)
          else:
               UpperCAmelCase__							: List[Any] 							=						torch.tensor(_lowercase       ,    device=_lowercase		)

          if is_buffer:
               UpperCAmelCase__							: List[Any] 							=						new_value
          else:
               UpperCAmelCase__							: Dict 							=						nn.Parameter(_lowercase       ,    requires_grad=old_value.requires_grad		)
               UpperCAmelCase__							: Dict 							=						new_value



def 							a__ (				lowerCAmelCase	:       str       ,    lowerCAmelCase	:       Tuple=None       ,    lowerCAmelCase	:       Optional[Any]=None       ,    lowerCAmelCase	:       List[str]=None       ,    lowerCAmelCase	:       str=False		):






     '''simple docstring'''
     for name, module in model.named_children():
          if current_key_name is None:
               UpperCAmelCase__							: List[str] 							=						[]
          current_key_name.append(_lowercase		)

          if (isinstance(_lowercase       ,    nn.Linear		) or isinstance(_lowercase       ,    _lowercase		)) and name not in modules_to_not_convert:
               # Check if the current key is not in the `modules_to_not_convert`
               if not any(key in ".".join(_lowercase		) for key in modules_to_not_convert		):
                    with init_empty_weights():
                         if isinstance(_lowercase       ,    _lowercase		):
                              UpperCAmelCase__							: Optional[Any] 							=						module.weight.shape
                         else:
                              UpperCAmelCase__							: Tuple 							=						module.in_features
                              UpperCAmelCase__							: Optional[int] 							=						module.out_features

                         if quantization_config.quantization_method() == "llm_int8":
                              UpperCAmelCase__							: str 							=						bnb.nn.LinearabitLt(
                                  _lowercase       ,    _lowercase       ,    module.bias is not None       ,    has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight       ,    threshold=quantization_config.llm_inta_threshold       ,    )
                              UpperCAmelCase__							: Optional[int] 							=						True
                         else:
                              if (
                                  quantization_config.llm_inta_skip_modules is not None
                                  and name in quantization_config.llm_inta_skip_modules
                              ):
                                   pass
                              else:
                                   UpperCAmelCase__							: List[Any] 							=						bnb.nn.Linearabit(
                                       _lowercase       ,    _lowercase       ,    module.bias is not None       ,    quantization_config.bnb_abit_compute_dtype       ,    compress_statistics=quantization_config.bnb_abit_use_double_quant       ,    quant_type=quantization_config.bnb_abit_quant_type       ,    )
                                   UpperCAmelCase__							: int 							=						True
                    # Store the module class in case we need to transpose the weight later
                         UpperCAmelCase__							: Union[str, Any] 							=						type(_lowercase		)
                         # Force requires grad to False to avoid unexpected errors
                         model._modules[name].requires_grad_(_lowercase		)
          if len(list(module.children()		)		) > 0:
               UpperCAmelCase__							: Union[str, Any] 							=						_replace_with_bnb_linear(
                   _lowercase       ,    _lowercase       ,    _lowercase       ,    _lowercase       ,    has_been_replaced=_lowercase       ,    )
          # Remove the last key for recursion
          current_key_name.pop(-1		)
     return model, has_been_replaced



def 							a__ (				lowerCAmelCase	:       int       ,    lowerCAmelCase	:       List[Any]=None       ,    lowerCAmelCase	:       List[Any]=None       ,    lowerCAmelCase	:       int=None		):






     '''simple docstring'''
     UpperCAmelCase__							: Union[str, Any] 							=						['''lm_head'''] if modules_to_not_convert is None else modules_to_not_convert
     UpperCAmelCase__							: Union[str, Any] 							=						_replace_with_bnb_linear(
         _lowercase       ,    _lowercase       ,    _lowercase       ,    _lowercase		)

     if not has_been_replaced:
          logger.warning(
              "You are loading your model in 8bit or 4bit but no linear modules were found in your model."
              " Please double check your model architecture, or submit an issue on github if you think this is"
              " a bug."		)

     return model



def 							a__ (				*lowerCAmelCase	:       List[str]       ,    **lowerCAmelCase	:       int		):






     '''simple docstring'''
     warnings.warn(
         "`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead"       ,    _lowercase       ,    )
     return replace_with_bnb_linear(*_lowercase       ,    **_lowercase		)



def 							a__ (				*lowerCAmelCase	:       int       ,    **lowerCAmelCase	:       List[Any]		):






     '''simple docstring'''
     warnings.warn(
         "`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead"       ,    _lowercase       ,    )
     return set_module_quantized_tensor_to_device(*_lowercase       ,    **_lowercase		)



def 							a__ (				lowerCAmelCase	:       List[Any]		):






     '''simple docstring'''
     UpperCAmelCase__							: Tuple 							=						deepcopy(_lowercase		)  # this has 0 cost since it is done inside `init_empty_weights` context manager`
     tied_model.tie_weights()

     UpperCAmelCase__							: str 							=						find_tied_parameters(_lowercase		)
     # For compatibility with Accelerate < 0.18
     if isinstance(_lowercase       ,    _lowercase		):
          UpperCAmelCase__							: Union[str, Any] 							=						sum(list(tied_params.values()		)       ,    []		) + list(tied_params.keys()		)
     else:
          UpperCAmelCase__							: Optional[int] 							=						sum(_lowercase       ,    []		)
     UpperCAmelCase__							: Optional[int] 							=						len(_lowercase		) > 0

     # Check if it is a base model
     UpperCAmelCase__							: List[Any] 							=						not hasattr(_lowercase       ,    model.base_model_prefix		)

     # Ignore this for base models (BertModel, GPT2Model, etc.)
     if (not has_tied_params) and is_base_model:
          return []

     # otherwise they have an attached head
     UpperCAmelCase__							: List[Any] 							=						list(model.named_children()		)
     UpperCAmelCase__							: Tuple 							=						[list_modules[-1][0]]

     # add last module together with tied weights
     UpperCAmelCase__							: Union[str, Any] 							=						set(_lowercase		) - set(_lowercase		)
     UpperCAmelCase__							: List[Any] 							=						list(set(_lowercase		)		) + list(_lowercase		)

     # remove ".weight" from the keys
     UpperCAmelCase__							: int 							=						['''.weight''', '''.bias''']
     UpperCAmelCase__							: Union[str, Any] 							=						[]
     for name in list_untouched:
          for name_to_remove in names_to_remove:
               if name_to_remove in name:
                    UpperCAmelCase__							: int 							=						name.replace(_lowercase       ,    ""		)
          filtered_module_names.append(_lowercase		)

     return filtered_module_names







"""simple docstring"""




import argparse

import torch

from transformers import GPTaConfig, GPTaModel, load_tf_weights_in_gpta
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()



def 							a__ (				lowerCAmelCase	:       List[Any]       ,    lowerCAmelCase	:       Any       ,    lowerCAmelCase	:       List[Any]		):






     '''simple docstring'''
     # Construct model
     if gpta_config_file == "":
          UpperCAmelCase__							: Optional[int] 							=						GPTaConfig()
     else:
          UpperCAmelCase__							: Dict 							=						GPTaConfig.from_json_file(lowerCAmelCase		)
     UpperCAmelCase__							: Optional[Any] 							=						GPTaModel(lowerCAmelCase		)

     # Load weights from numpy
     load_tf_weights_in_gpta(lowerCAmelCase       ,    lowerCAmelCase       ,    lowerCAmelCase		)

     # Save pytorch-model
     UpperCAmelCase__							: Optional[int] 							=						pytorch_dump_folder_path + "/" + WEIGHTS_NAME
     UpperCAmelCase__							: Any 							=						pytorch_dump_folder_path + "/" + CONFIG_NAME
     print(F"Save PyTorch model to {pytorch_weights_dump_path}"		)
     torch.save(model.state_dict()       ,    lowerCAmelCase		)
     print(F"Save configuration file to {pytorch_config_dump_path}"		)
     with open(lowerCAmelCase       ,    "w"       ,    encoding="utf-8"		) as f:
          f.write(config.to_json_string()		)


if __name__ == "__main__":
    A__      :	Optional[int]          =    argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        """--gpt2_checkpoint_path""", default=None, type=str, required=True, help="""Path to the TensorFlow checkpoint path."""
    )
    parser.add_argument(
        """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
    )
    parser.add_argument(
        """--gpt2_config_file""",
        default="""""",
        type=str,
        help=(
            """An optional config json file corresponding to the pre-trained OpenAI model. \n"""
            """This specifies the model architecture."""
        ),
    )
    A__      :	Optional[Any]          =    parser.parse_args()
    convert_gpta_checkpoint_to_pytorch(args.gpta_checkpoint_path, args.gpta_config_file, args.pytorch_dump_folder_path)



