


'''simple docstring'''




import inspect
import os
import unittest
from pathlib import Path

import torch

import accelerate
from accelerate.test_utils import execute_subprocess_async
from accelerate.test_utils.testing import run_command



class  lowerCAmelCase				(				unittest.TestCase							):
	_lowerCamelCase   :					Any			      = inspect.getfile(accelerate.test_utils							)
	_lowerCamelCase   :					Any			      = os.path.sep.join(mod_file.split(os.path.sep							)[:-1] + ["""scripts""", """test_cli.py"""]							)

	_lowerCamelCase   :					Optional[int]			      = ["""accelerate""", """launch"""]
	_lowerCamelCase   :					int			      = Path.home() / """.cache/huggingface/accelerate"""
	_lowerCamelCase   :					List[Any]			      = """default_config.yaml"""
	_lowerCamelCase   :					Any			      = config_folder / config_file
	_lowerCamelCase   :					List[str]			      = config_folder / """_default_config.yaml"""

	_lowerCamelCase   :					int			      = Path("""tests/test_configs"""							)



	@classmethod
	def 				lowercase     (					cls    ):
		if cls.config_path.is_file():
			cls.config_path.rename(cls.changed_path    )



	@classmethod
	def 				lowercase     (					cls    ):
		if cls.changed_path.is_file():
			cls.changed_path.rename(cls.config_path    )



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[Any]      							= self.base_cmd
		if torch.cuda.is_available() and (torch.cuda.device_count() > 1):
			cmd += ["--multi_gpu"]
		execute_subprocess_async(cmd + [self.test_file_path] ,					env=os.environ.copy()    )



	def 				lowercase     (					self    ):
		for config in sorted(self.test_config_path.glob('**/*.yaml'    )    ):
			with self.subTest(config_file=snake_case__    ):
				execute_subprocess_async(
				    self.base_cmd + ['--config_file', str(snake_case__    ), self.test_file_path] ,					env=os.environ.copy()    )



	def 				lowercase     (					self    ):
		execute_subprocess_async(['accelerate', 'test'] ,					env=os.environ.copy()    )



class  lowerCAmelCase				(				unittest.TestCase							):
	_lowerCamelCase   :					int			      = """test-tpu"""
	_lowerCamelCase   :					Tuple			      = """us-central1-a"""
	_lowerCamelCase   :					Optional[int]			      = """ls"""
	_lowerCamelCase   :					Union[str, Any]			      = ["""accelerate""", """tpu-config"""]
	_lowerCamelCase   :					Optional[int]			      = """cd /usr/share"""
	_lowerCamelCase   :					List[str]			      = """tests/test_samples/test_command_file.sh"""
	_lowerCamelCase   :					Any			      = """Running gcloud compute tpus tpu-vm ssh"""



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[int]      							= run_command(
		    self.cmd
		    + ['--command', self.command, '--tpu_zone', self.tpu_zone, '--tpu_name', self.tpu_name, '--debug'] ,					return_stdout=snake_case__ ,					)
		self.assertIn(
		    f"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all" ,					snake_case__ ,					)



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Tuple      							= run_command(
		    self.cmd
		    + [
		        '--config_file',
		        'tests/test_configs/0_12_0.yaml',
		        '--command',
		        self.command,
		        '--tpu_zone',
		        self.tpu_zone,
		        '--tpu_name',
		        self.tpu_name,
		        '--debug',
		    ] ,					return_stdout=snake_case__ ,					)
		self.assertIn(
		    f"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all" ,					snake_case__ ,					)



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    int      							= run_command(
		    self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--debug'] ,					return_stdout=snake_case__    )
		self.assertIn(
		    f"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all" ,					snake_case__ ,					)



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Tuple      							= run_command(
		    self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--command', self.command, '--debug'] ,					return_stdout=snake_case__ ,					)
		self.assertIn(
		    f"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls --worker all" ,					snake_case__ ,					)



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[Any]      							= run_command(
		    self.cmd
		    + [
		        '--config_file',
		        'tests/test_configs/latest.yaml',
		        '--command',
		        self.command,
		        '--command',
		        'echo "Hello World"',
		        '--debug',
		    ] ,					return_stdout=snake_case__ ,					)
		self.assertIn(
		    f"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; ls; echo \"Hello World\" --worker all" ,					snake_case__ ,					)



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Dict      							= run_command(
		    self.cmd
		    + ['--config_file', 'tests/test_configs/latest.yaml', '--command_file', self.command_file, '--debug'] ,					return_stdout=snake_case__ ,					)
		self.assertIn(
		    f"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all" ,					snake_case__ ,					)



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    List[Any]      							= run_command(
		    self.cmd
		    + [
		        '--config_file',
		        'tests/test_configs/0_12_0.yaml',
		        '--command_file',
		        self.command_file,
		        '--tpu_zone',
		        self.tpu_zone,
		        '--tpu_name',
		        self.tpu_name,
		        '--debug',
		    ] ,					return_stdout=snake_case__ ,					)
		self.assertIn(
		    f"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; echo \"hello world\"; echo \"this is a second command\" --worker all" ,					snake_case__ ,					)



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    List[Any]      							= run_command(
		    self.cmd + ['--config_file', 'tests/test_configs/latest.yaml', '--install_accelerate', '--debug'] ,					return_stdout=snake_case__ ,					)
		self.assertIn(
		    f"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate -U; echo \"hello world\"; echo \"this is a second command\" --worker all" ,					snake_case__ ,					)



	def 				lowercase     (					self    ):
		lowerCAmelCase					:    Optional[Any]      							= run_command(
		    self.cmd
		    + [
		        '--config_file',
		        'tests/test_configs/latest.yaml',
		        '--install_accelerate',
		        '--accelerate_version',
		        '12.0.0',
		        '--debug',
		    ] ,					return_stdout=snake_case__ ,					)
		self.assertIn(
		    f"{self.gcloud} test-tpu --zone us-central1-a --command {self.base_output}; pip install accelerate==12.0.0; echo \"hello world\"; echo \"this is a second command\" --worker all" ,					snake_case__ ,					)




'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCAmelCase  :		str		     =	logging.get_logger(__name__)

_lowerCAmelCase  :		Tuple		     =	{
    's-JoL/Open-Llama-V1': 'https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json',
}



class  lowerCAmelCase				(				a							):
	_lowerCamelCase   :					Union[str, Any]			      = """open-llama"""



	def __init__(					self ,					snake_case__=10_0000 ,					snake_case__=4096 ,					snake_case__=1_1008 ,					snake_case__=32 ,					snake_case__=32 ,					snake_case__="silu" ,					snake_case__=2048 ,					snake_case__=0.0_2 ,					snake_case__=1e-6 ,					snake_case__=True ,					snake_case__=0 ,					snake_case__=1 ,					snake_case__=2 ,					snake_case__=False ,					snake_case__=True ,					snake_case__=0.1 ,					snake_case__=0.1 ,					snake_case__=True ,					snake_case__=True ,					snake_case__=None ,					**snake_case__ ,					):
		lowerCAmelCase					:    Tuple      							= vocab_size
		lowerCAmelCase					:    Optional[Any]      							= max_position_embeddings
		lowerCAmelCase					:    List[Any]      							= hidden_size
		lowerCAmelCase					:    List[Any]      							= intermediate_size
		lowerCAmelCase					:    Tuple      							= num_hidden_layers
		lowerCAmelCase					:    List[Any]      							= num_attention_heads
		lowerCAmelCase					:    List[Any]      							= hidden_act
		lowerCAmelCase					:    Union[str, Any]      							= initializer_range
		lowerCAmelCase					:    str      							= rms_norm_eps
		lowerCAmelCase					:    Optional[int]      							= use_cache
		lowerCAmelCase					:    Dict      							= kwargs.pop(
		    'use_memorry_efficient_attention' ,					snake_case__    )
		lowerCAmelCase					:    Optional[int]      							= hidden_dropout_prob
		lowerCAmelCase					:    Optional[Any]      							= attention_dropout_prob
		lowerCAmelCase					:    Union[str, Any]      							= use_stable_embedding
		lowerCAmelCase					:    Tuple      							= shared_input_output_embedding
		lowerCAmelCase					:    Tuple      							= rope_scaling
		self._rope_scaling_validation()

		super().__init__(
		    pad_token_id=snake_case__ ,					bos_token_id=snake_case__ ,					eos_token_id=snake_case__ ,					tie_word_embeddings=snake_case__ ,					**snake_case__ ,					)



	def 				lowercase     (					self    ):
		if self.rope_scaling is None:
			return

		if not isinstance(self.rope_scaling ,					snake_case__    ) or len(self.rope_scaling    ) != 2:
			raise ValueError(
			    '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
			    f"got {self.rope_scaling}"    )
		lowerCAmelCase					:    List[Any]      							= self.rope_scaling.get('type' ,					snake_case__    )
		lowerCAmelCase					:    List[str]      							= self.rope_scaling.get('factor' ,					snake_case__    )
		if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
			raise ValueError(
			    f"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"    )
		if rope_scaling_factor is None or not isinstance(snake_case__ ,					snake_case__    ) or rope_scaling_factor <= 1.0:
			raise ValueError(f"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"    )
