




# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
 import botoa  # noqa: F401

def     _SCREAMING_SNAKE_CASE    (						lowercase			: List[Any]  ):


      '''simple docstring'''



      lowerCamelCase_	  =					botoa.client('iam'  )

      lowerCamelCase_	  =					{
          "Version": "2012-10-17",
          "Statement": [
              {"Effect": "Allow", "Principal": {"Service": "sagemaker.amazonaws.com"}, "Action": "sts:AssumeRole"}
          ],
      }
      try:
            # create the role, associated with the chosen trust policy
            iam_client.create_role(
                RoleName=_lowerCAmelCase			,							AssumeRolePolicyDocument=json.dumps(_lowerCAmelCase			,							indent=2  )  )
            lowerCamelCase_	  =					{
                "Version": "2012-10-17",
                "Statement": [
                    {
                        "Effect": "Allow",
                        "Action": [
                            "sagemaker:*",
                            "ecr:GetDownloadUrlForLayer",
                            "ecr:BatchGetImage",
                            "ecr:BatchCheckLayerAvailability",
                            "ecr:GetAuthorizationToken",
                            "cloudwatch:PutMetricData",
                            "cloudwatch:GetMetricData",
                            "cloudwatch:GetMetricStatistics",
                            "cloudwatch:ListMetrics",
                            "logs:CreateLogGroup",
                            "logs:CreateLogStream",
                            "logs:DescribeLogStreams",
                            "logs:PutLogEvents",
                            "logs:GetLogEvents",
                            "s3:CreateBucket",
                            "s3:ListBucket",
                            "s3:GetBucketLocation",
                            "s3:GetObject",
                            "s3:PutObject",
                        ],
                        "Resource": "*",
                    }
                ],
            }
            # attach policy to role
            iam_client.put_role_policy(
                RoleName=_lowerCAmelCase			,							PolicyName=f"""{role_name}_policy_permission"""			,							PolicyDocument=json.dumps(_lowerCAmelCase			,							indent=2  )			,							)
      except iam_client.exceptions.EntityAlreadyExistsException:
            print(f"""role {role_name} already exists. Using existing one"""  )

def     _SCREAMING_SNAKE_CASE    (						lowercase			: Optional[Any]  ):


      '''simple docstring'''



      lowerCamelCase_	  =					botoa.client('iam'  )
      return iam_client.get_role(RoleName=_lowerCAmelCase  )["Role"]["Arn"]






def     _SCREAMING_SNAKE_CASE    (						):


      '''simple docstring'''



      lowerCamelCase_	  =					_ask_options(
          'How do you want to authorize?'			,							['AWS Profile', 'Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) ']			,							_lowerCAmelCase			,							)
      lowerCamelCase_	  =					None
      if credentials_configuration == 0:
            lowerCamelCase_	  =					_ask_field('Enter your AWS Profile name: [default] '			,							default='default'  )
            lowerCamelCase_	  =					aws_profile
      else:
            print(
                'Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,'
                '`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`'  )
            lowerCamelCase_	  =					_ask_field('AWS Access Key ID: '  )
            lowerCamelCase_	  =					aws_access_key_id

            lowerCamelCase_	  =					_ask_field('AWS Secret Access Key: '  )
            lowerCamelCase_	  =					aws_secret_access_key

      lowerCamelCase_	  =					_ask_field('Enter your AWS Region: [us-east-1]'			,							default='us-east-1'  )
      lowerCamelCase_	  =					aws_region

      lowerCamelCase_	  =					_ask_options(
          'Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?'			,							['Provide IAM Role name', 'Create new IAM role using credentials']			,							_lowerCAmelCase			,							)
      if role_management == 0:
            lowerCamelCase_	  =					_ask_field('Enter your IAM role name: '  )
      else:
            lowerCamelCase_	  =					"accelerate_sagemaker_execution_role"
            print(f"""Accelerate will create an iam role \"{iam_role_name}\" using the provided credentials"""  )
            _create_iam_role_for_sagemaker(_lowerCAmelCase  )

      lowerCamelCase_	  =					_ask_field(
          'Do you want to use custom Docker image? [yes/NO]: '			,							_convert_yes_no_to_bool			,							default=_lowerCAmelCase			,							error_message='Please enter yes or no.'			,							)
      lowerCamelCase_	  =					None
      if is_custom_docker_image:
            lowerCamelCase_	  =					_ask_field('Enter your Docker image: '			,							lambda lowercase  : str(_lowerCAmelCase  ).lower()  )

      lowerCamelCase_	  =					_ask_field(
          'Do you want to provide SageMaker input channels with data locations? [yes/NO]: '			,							_convert_yes_no_to_bool			,							default=_lowerCAmelCase			,							error_message='Please enter yes or no.'			,							)
      lowerCamelCase_	  =					None
      if is_sagemaker_inputs_enabled:
            lowerCamelCase_	  =					_ask_field(
                'Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): '			,							lambda lowercase  : str(_lowerCAmelCase  ).lower()			,							)

      lowerCamelCase_	  =					_ask_field(
          'Do you want to enable SageMaker metrics? [yes/NO]: '			,							_convert_yes_no_to_bool			,							default=_lowerCAmelCase			,							error_message='Please enter yes or no.'			,							)
      lowerCamelCase_	  =					None
      if is_sagemaker_metrics_enabled:
            lowerCamelCase_	  =					_ask_field(
                'Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): '			,							lambda lowercase  : str(_lowerCAmelCase  ).lower()			,							)

      lowerCamelCase_	  =					_ask_options(
          'What is the distributed mode?'			,							['No distributed training', 'Data parallelism']			,							_convert_sagemaker_distributed_mode			,							)
      lowerCamelCase_	  =					{}
      lowerCamelCase_	  =					_ask_field(
          'Do you wish to optimize your script with torch dynamo?[yes/NO]:'			,							_convert_yes_no_to_bool			,							default=_lowerCAmelCase			,							error_message='Please enter yes or no.'			,							)
      if use_dynamo:
            lowerCamelCase_	  =					"dynamo_"
            lowerCamelCase_	  =					_ask_options(
                'Which dynamo backend would you like to use?'			,							[x.lower() for x in DYNAMO_BACKENDS]			,							_convert_dynamo_backend			,							default=2			,							)
            lowerCamelCase_	  =					_ask_field(
                'Do you want to customize the defaults sent to torch.compile? [yes/NO]: '			,							_convert_yes_no_to_bool			,							default=_lowerCAmelCase			,							error_message='Please enter yes or no.'			,							)

            if use_custom_options:
                  lowerCamelCase_	  =					_ask_options(
                      'Which mode do you want to use?'			,							_lowerCAmelCase			,							lambda lowercase  : TORCH_DYNAMO_MODES[int(_lowerCAmelCase  )]			,							default='default'			,							)
                  lowerCamelCase_	  =					_ask_field(
                      'Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: '			,							_convert_yes_no_to_bool			,							default=_lowerCAmelCase			,							error_message='Please enter yes or no.'			,							)
                  lowerCamelCase_	  =					_ask_field(
                      'Do you want to enable dynamic shape tracing? [yes/NO]: '			,							_convert_yes_no_to_bool			,							default=_lowerCAmelCase			,							error_message='Please enter yes or no.'			,							)
      lowerCamelCase_	  =					"Which EC2 instance type you want to use for your training?"
      if distributed_type != SageMakerDistributedType.NO:
            lowerCamelCase_	  =					_ask_options(
                _lowerCAmelCase			,							_lowerCAmelCase			,							lambda lowercase  : SAGEMAKER_PARALLEL_EC2_INSTANCES[int(_lowerCAmelCase  )]  )
      else:
            eca_instance_query += "? [ml.p3.2xlarge]:"
            lowerCamelCase_	  =					_ask_field(_lowerCAmelCase			,							lambda lowercase  : str(_lowerCAmelCase  ).lower()			,							default='ml.p3.2xlarge'  )

      lowerCamelCase_	  =					1
      if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
            lowerCamelCase_	  =					_ask_field(
                'How many machines do you want use? [1]: '			,							_lowerCAmelCase			,							default=1			,							)

      lowerCamelCase_	  =					_ask_options(
          'Do you wish to use FP16 or BF16 (mixed precision)?'			,							['no', 'fp16', 'bf16', 'fp8']			,							_convert_mixed_precision			,							)

      if use_dynamo and mixed_precision == "no":
            print(
                'Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts.'  )

      return SageMakerConfig(
          image_uri=_lowerCAmelCase			,							compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER			,							distributed_type=_lowerCAmelCase			,							use_cpu=_lowerCAmelCase			,							dynamo_config=_lowerCAmelCase			,							eca_instance_type=_lowerCAmelCase			,							profile=_lowerCAmelCase			,							region=_lowerCAmelCase			,							iam_role_name=_lowerCAmelCase			,							mixed_precision=_lowerCAmelCase			,							num_machines=_lowerCAmelCase			,							sagemaker_inputs_file=_lowerCAmelCase			,							sagemaker_metrics_file=_lowerCAmelCase			,							)









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase       :	str									=     logging.get_logger(__name__)

lowerCamelCase       :	List[str]									=     {
    "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json",
}







class 						A(		UpperCamelCase     ):



 '''simple docstring'''


 UpperCamelCase		     =		'''gpt_neox_japanese'''

 def __init__(       self      :						int		,							A_      :						Dict=32000		,							A_      :						List[Any]=2560		,							A_      :						Dict=32		,							A_      :						Union[str, Any]=32		,							A_      :						List[Any]=4		,							A_      :						List[str]="gelu"		,							A_      :						Dict=1.00		,							A_      :						int=10000		,							A_      :						Dict=2048		,							A_      :						Dict=0.02		,							A_      :						Any=1E-5		,							A_      :						Union[str, Any]=True		,							A_      :						int=31996		,							A_      :						List[str]=31999		,							A_      :						List[Any]=0.1		,							A_      :						List[Any]=0.0		,							**A_      :						Tuple		,							)     ->			Dict:

       """simple docstring"""






       super().__init__(bos_token_id=A_		,							eos_token_id=A_		,							**A_						)
       lowerCamelCase_	  =					vocab_size
       lowerCamelCase_	  =					max_position_embeddings
       lowerCamelCase_	  =					hidden_size
       lowerCamelCase_	  =					num_hidden_layers
       lowerCamelCase_	  =					num_attention_heads
       lowerCamelCase_	  =					intermediate_multiple_size
       lowerCamelCase_	  =					hidden_act
       lowerCamelCase_	  =					rotary_pct
       lowerCamelCase_	  =					rotary_emb_base
       lowerCamelCase_	  =					initializer_range
       lowerCamelCase_	  =					layer_norm_eps
       lowerCamelCase_	  =					use_cache
       lowerCamelCase_	  =					attention_dropout
       lowerCamelCase_	  =					hidden_dropout



