





from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowerCAmelCase_						   =    logging.get_logger(__name__)

lowerCAmelCase_						   =    {
    'EleutherAI/gpt-j-6B': 'https://huggingface.co/EleutherAI/gpt-j-6B/resolve/main/config.json',
    # See all GPT-J models at https://huggingface.co/models?filter=gpt_j
}



class        _lowerCAmelCase    (			UpperCamelCase_							):


  '''simple docstring'''
  a_							:       str      					="""gptj"""
  a_							:       Optional[Any]      					={
      """max_position_embeddings""": """n_positions""",
      """hidden_size""": """n_embd""",
      """num_attention_heads""": """n_head""",
      """num_hidden_layers""": """n_layer""",
  }







  def __init__(      self      :					Union[str, Any]      ,				UpperCamelCase      :					Optional[Any]=5_04_00      ,				UpperCamelCase      :					Tuple=20_48      ,				UpperCamelCase      :					Optional[int]=40_96      ,				UpperCamelCase      :					List[str]=28      ,				UpperCamelCase      :					List[Any]=16      ,				UpperCamelCase      :					str=64      ,				UpperCamelCase      :					str=None      ,				UpperCamelCase      :					Tuple="gelu_new"      ,				UpperCamelCase      :					List[str]=0.0      ,				UpperCamelCase      :					Any=0.0      ,				UpperCamelCase      :					List[Any]=0.0      ,				UpperCamelCase      :					Optional[Any]=1e-5      ,				UpperCamelCase      :					str=0.02      ,				UpperCamelCase      :					List[Any]=True      ,				UpperCamelCase      :					Dict=5_02_56      ,				UpperCamelCase      :					Any=5_02_56      ,				UpperCamelCase      :					int=False      ,				**UpperCamelCase      :					str      ,				):


    '''simple docstring'''





    _snake_case   :				int														=	vocab_size
    _snake_case   :				List[str]														=	n_positions
    _snake_case   :				int														=	n_embd
    _snake_case   :				Tuple														=	n_layer
    _snake_case   :				Any														=	n_head
    _snake_case   :				Union[str, Any]														=	n_inner
    _snake_case   :				List[Any]														=	rotary_dim
    _snake_case   :				Optional[int]														=	activation_function
    _snake_case   :				Dict														=	resid_pdrop
    _snake_case   :				Dict														=	embd_pdrop
    _snake_case   :				Dict														=	attn_pdrop
    _snake_case   :				Any														=	layer_norm_epsilon
    _snake_case   :				int														=	initializer_range
    _snake_case   :				str														=	use_cache

    _snake_case   :				Optional[int]														=	bos_token_id
    _snake_case   :				str														=	eos_token_id

    super().__init__(
        bos_token_id=UpperCamelCase__      ,				eos_token_id=UpperCamelCase__      ,				tie_word_embeddings=UpperCamelCase__      ,				**UpperCamelCase__							)




class        _lowerCAmelCase    (			UpperCamelCase_							):


  '''simple docstring'''







  def __init__(      self      :					Optional[Any]      ,				UpperCamelCase      :					PretrainedConfig      ,				UpperCamelCase      :					str = "default"      ,				UpperCamelCase      :					List[PatchingSpec] = None      ,				UpperCamelCase      :					bool = False      ,				):


    '''simple docstring'''





    super().__init__(UpperCamelCase__      ,				task=UpperCamelCase__      ,				patching_specs=UpperCamelCase__      ,				use_past=UpperCamelCase__							)
    if not getattr(self._config      ,				'pad_token_id'      ,				UpperCamelCase__							):
      # TODO: how to do that better?
      _snake_case   :				Any														=	0







  @property
  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    _snake_case   :				str														=	OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}							)
    if self.use_past:
      self.fill_with_past_key_values_(UpperCamelCase__      ,				direction='inputs'							)
      _snake_case   :				Optional[Any]														=	{0: '''batch''', 1: '''past_sequence + sequence'''}
    else:
      _snake_case   :				List[str]														=	{0: '''batch''', 1: '''sequence'''}

    return common_inputs







  @property
  def UpperCamelCase_ (      self      :					Union[str, Any]							):


    '''simple docstring'''





    return self._config.n_layer







  @property
  def UpperCamelCase_ (      self      :					Union[str, Any]							):


    '''simple docstring'''





    return self._config.n_head







  def UpperCamelCase_ (      self      :					Optional[int]      ,				UpperCamelCase      :					PreTrainedTokenizer      ,				UpperCamelCase      :					int = -1      ,				UpperCamelCase      :					int = -1      ,				UpperCamelCase      :					bool = False      ,				UpperCamelCase      :					Optional[TensorType] = None      ,				):


    '''simple docstring'''





    _snake_case   :				List[str]														=	super(UpperCamelCase__      ,				self							).generate_dummy_inputs(
        UpperCamelCase__      ,				batch_size=UpperCamelCase__      ,				seq_length=UpperCamelCase__      ,				is_pair=UpperCamelCase__      ,				framework=UpperCamelCase__							)

    # We need to order the input in the way they appears in the forward()
    _snake_case   :				Any														=	OrderedDict({'input_ids': common_inputs['input_ids']}							)

    # Need to add the past_keys
    if self.use_past:
      if not is_torch_available():
        raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'							)
      else:
        import torch

        _snake_case   :				Optional[Any]														=	common_inputs['''input_ids'''].shape
        # Not using the same length for past_key_values
        _snake_case   :				int														=	seqlen + 2
        _snake_case   :				Optional[int]														=	(
            batch,
            self.num_attention_heads,
            past_key_values_length,
            self._config.hidden_size // self.num_attention_heads,
        )
        _snake_case   :				Union[str, Any]														=	[
            (torch.zeros(UpperCamelCase__							), torch.zeros(UpperCamelCase__							)) for _ in range(self.num_layers							)
        ]

    _snake_case   :				str														=	common_inputs['''attention_mask''']
    if self.use_past:
      _snake_case   :				List[Any]														=	ordered_inputs['''attention_mask'''].dtype
      _snake_case   :				int														=	torch.cat(
          [ordered_inputs['attention_mask'], torch.ones(UpperCamelCase__      ,				UpperCamelCase__      ,				dtype=UpperCamelCase__							)]      ,				dim=1							)

    return ordered_inputs







  @property
  def UpperCamelCase_ (      self      :					List[Any]							):


    '''simple docstring'''





    return 13











# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


lowerCAmelCase_						   =    """Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine"""


def     lowerCamelCase_	(      )->		Tuple:
  _snake_case   :				int														=	_ask_options(
      'In which compute environment are you running?'    ,							['This machine', 'AWS (Amazon SageMaker)']    ,							_convert_compute_environment    ,							)
  if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
    _snake_case   :				int														=	get_sagemaker_input()
  else:
    _snake_case   :				Any														=	get_cluster_input()
  return config


def     lowerCamelCase_	(      lowerCAmelCase:	str=None				)->		Any:
  if subparsers is not None:
    _snake_case   :				List[Any]														=	subparsers.add_parser('config'    ,							description=lowerCAmelCase				)
  else:
    _snake_case   :				Dict														=	argparse.ArgumentParser('Accelerate config command'    ,							description=lowerCAmelCase				)

  parser.add_argument(
      '--config_file'    ,							default=lowerCAmelCase    ,							help=(
          'The path to use to store the config file. Will default to a file named default_config.yaml in the cache '
          'location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '
          'such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '
          'with \'huggingface\'.'
      )    ,							)

  if subparsers is not None:
    parser.set_defaults(func=lowerCAmelCase				)
  return parser


def     lowerCamelCase_	(      lowerCAmelCase:	Any				)->		Any:
  _snake_case   :				Dict														=	get_user_input()
  if args.config_file is not None:
    _snake_case   :				List[str]														=	args.config_file
  else:
    if not os.path.isdir(lowerCAmelCase				):
      os.makedirs(lowerCAmelCase				)
    _snake_case   :				Union[str, Any]														=	default_yaml_config_file

  if config_file.endswith('.json'				):
    config.to_json_file(lowerCAmelCase				)
  else:
    config.to_yaml_file(lowerCAmelCase				)
  print(F"""accelerate configuration saved at {config_file}"""				)


def     lowerCamelCase_	(      )->		Dict:
  _snake_case   :				List[str]														=	config_command_parser()
  _snake_case   :				str														=	parser.parse_args()
  config_command(lowerCAmelCase				)


if __name__ == "__main__":
      main()




