






'''simple docstring'''




import unittest

from transformers import BertGenerationConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import BertGenerationDecoder, BertGenerationEncoder







class    A_	:
							"""simple docstring"""




							def __init__(		self			:Optional[Any]       ,							lowerCAmelCase__			:List[Any]       ,							lowerCAmelCase__			:Any=13       ,							lowerCAmelCase__			:Optional[Any]=7       ,							lowerCAmelCase__			:int=True       ,							lowerCAmelCase__			:Tuple=True       ,							lowerCAmelCase__			:List[Any]=99       ,							lowerCAmelCase__			:List[str]=32       ,							lowerCAmelCase__			:int=5       ,							lowerCAmelCase__			:Union[str, Any]=4       ,							lowerCAmelCase__			:Union[str, Any]=37       ,							lowerCAmelCase__			:Optional[int]="gelu"       ,							lowerCAmelCase__			:List[str]=0.1       ,							lowerCAmelCase__			:int=0.1       ,							lowerCAmelCase__			:List[str]=50       ,							lowerCAmelCase__			:int=0.0_2       ,							lowerCAmelCase__			:Dict=True       ,							lowerCAmelCase__			:List[str]=None       ,							)				->  Union[str, Any]:


										'''simple docstring'''




										snake_case_				:					List[Any]   =       parent
										snake_case_				:					Any   =       batch_size
										snake_case_				:					int   =       seq_length
										snake_case_				:					Dict   =       is_training
										snake_case_				:					str   =       use_input_mask
										snake_case_				:					Any   =       vocab_size
										snake_case_				:					List[str]   =       hidden_size
										snake_case_				:					Tuple   =       num_hidden_layers
										snake_case_				:					Optional[int]   =       num_attention_heads
										snake_case_				:					List[Any]   =       intermediate_size
										snake_case_				:					Optional[int]   =       hidden_act
										snake_case_				:					Tuple   =       hidden_dropout_prob
										snake_case_				:					Optional[Any]   =       attention_probs_dropout_prob
										snake_case_				:					Union[str, Any]   =       max_position_embeddings
										snake_case_				:					str   =       initializer_range
										snake_case_				:					Dict   =       use_labels
										snake_case_				:					List[str]   =       scope

							def 							_A	(		self			:List[Any]				)				->  Union[str, Any]:


										'''simple docstring'''




										snake_case_				:					List[Any]   =       ids_tensor([self.batch_size, self.seq_length]       ,							self.vocab_size				)

										snake_case_				:					List[Any]   =       None
										if self.use_input_mask:
													snake_case_				:					str   =       random_attention_mask([self.batch_size, self.seq_length]				)

										if self.use_labels:
													snake_case_				:					int   =       ids_tensor([self.batch_size, self.seq_length]       ,							self.vocab_size				)

										snake_case_				:					Any   =       self.get_config()

										return config, input_ids, input_mask, token_labels

							def 							_A	(		self			:str				)				->  List[str]:


										'''simple docstring'''




										return BertGenerationConfig(
										    vocab_size=self.vocab_size       ,							hidden_size=self.hidden_size       ,							num_hidden_layers=self.num_hidden_layers       ,							num_attention_heads=self.num_attention_heads       ,							intermediate_size=self.intermediate_size       ,							hidden_act=self.hidden_act       ,							hidden_dropout_prob=self.hidden_dropout_prob       ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,							max_position_embeddings=self.max_position_embeddings       ,							is_decoder=lowerCAmelCase__       ,							initializer_range=self.initializer_range       ,							)

							def 							_A	(		self			:Optional[Any]				)				->  Union[str, Any]:


										'''simple docstring'''




										(
										    (
										    snake_case_
										),	(
										    snake_case_
										),	(
										    snake_case_
										),	(
										    snake_case_
										),	
										)				:					List[str]   =       self.prepare_config_and_inputs()

										snake_case_				:					Any   =       True
										snake_case_				:					Optional[Any]   =       floats_tensor([self.batch_size, self.seq_length, self.hidden_size]				)
										snake_case_				:					Optional[Any]   =       ids_tensor([self.batch_size, self.seq_length]       ,							vocab_size=2				)

										return (
										    config,
										    input_ids,
										    input_mask,
										    token_labels,
										    encoder_hidden_states,
										    encoder_attention_mask,
										)

							def 							_A	(		self			:Tuple       ,							lowerCAmelCase__			:Tuple       ,							lowerCAmelCase__			:List[str]       ,							lowerCAmelCase__			:Optional[Any]       ,							lowerCAmelCase__			:Tuple       ,							**lowerCAmelCase__			:Dict       ,							)				->  Optional[Any]:


										'''simple docstring'''




										snake_case_				:					Optional[int]   =       BertGenerationEncoder(config=lowerCAmelCase__				)
										model.to(lowerCAmelCase__				)
										model.eval()
										snake_case_				:					int   =       model(lowerCAmelCase__       ,							attention_mask=lowerCAmelCase__				)
										snake_case_				:					Any   =       model(lowerCAmelCase__				)
										self.parent.assertEqual(result.last_hidden_state.shape       ,							(self.batch_size, self.seq_length, self.hidden_size)				)

							def 							_A	(		self			:str       ,							lowerCAmelCase__			:Dict       ,							lowerCAmelCase__			:Optional[Any]       ,							lowerCAmelCase__			:Union[str, Any]       ,							lowerCAmelCase__			:str       ,							lowerCAmelCase__			:List[Any]       ,							lowerCAmelCase__			:Optional[Any]       ,							**lowerCAmelCase__			:Optional[Any]       ,							)				->  int:


										'''simple docstring'''




										snake_case_				:					str   =       True
										snake_case_				:					Tuple   =       BertGenerationEncoder(config=lowerCAmelCase__				)
										model.to(lowerCAmelCase__				)
										model.eval()
										snake_case_				:					List[str]   =       model(
										    lowerCAmelCase__       ,							attention_mask=lowerCAmelCase__       ,							encoder_hidden_states=lowerCAmelCase__       ,							encoder_attention_mask=lowerCAmelCase__       ,							)
										snake_case_				:					List[str]   =       model(
										    lowerCAmelCase__       ,							attention_mask=lowerCAmelCase__       ,							encoder_hidden_states=lowerCAmelCase__       ,							)
										self.parent.assertEqual(result.last_hidden_state.shape       ,							(self.batch_size, self.seq_length, self.hidden_size)				)

							def 							_A	(		self			:Union[str, Any]       ,							lowerCAmelCase__			:str       ,							lowerCAmelCase__			:int       ,							lowerCAmelCase__			:Any       ,							lowerCAmelCase__			:List[Any]       ,							lowerCAmelCase__			:str       ,							lowerCAmelCase__			:Optional[int]       ,							**lowerCAmelCase__			:int       ,							)				->  Tuple:


										'''simple docstring'''




										snake_case_				:					Tuple   =       True
										snake_case_				:					str   =       True
										snake_case_				:					Union[str, Any]   =       BertGenerationDecoder(config=lowerCAmelCase__				).to(lowerCAmelCase__				).eval()

										# first forward pass
										snake_case_				:					Dict   =       model(
										    lowerCAmelCase__       ,							attention_mask=lowerCAmelCase__       ,							encoder_hidden_states=lowerCAmelCase__       ,							encoder_attention_mask=lowerCAmelCase__       ,							use_cache=lowerCAmelCase__       ,							)
										snake_case_				:					List[str]   =       outputs.past_key_values

										# create hypothetical multiple next token and extent to next_input_ids
										snake_case_				:					Union[str, Any]   =       ids_tensor((self.batch_size, 3)       ,							config.vocab_size				)
										snake_case_				:					List[str]   =       ids_tensor((self.batch_size, 3)       ,							vocab_size=2				)

										# append to next input_ids and
										snake_case_				:					List[str]   =       torch.cat([input_ids, next_tokens]       ,							dim=-1				)
										snake_case_				:					str   =       torch.cat([input_mask, next_mask]       ,							dim=-1				)

										snake_case_				:					str   =       model(
										    lowerCAmelCase__       ,							attention_mask=lowerCAmelCase__       ,							encoder_hidden_states=lowerCAmelCase__       ,							encoder_attention_mask=lowerCAmelCase__       ,							output_hidden_states=lowerCAmelCase__       ,							)["hidden_states"][0]
										snake_case_				:					List[Any]   =       model(
										    lowerCAmelCase__       ,							attention_mask=lowerCAmelCase__       ,							encoder_hidden_states=lowerCAmelCase__       ,							encoder_attention_mask=lowerCAmelCase__       ,							past_key_values=lowerCAmelCase__       ,							output_hidden_states=lowerCAmelCase__       ,							)["hidden_states"][0]

										# select random slice
										snake_case_				:					List[str]   =       ids_tensor((1,)       ,							output_from_past.shape[-1]				).item()
										snake_case_				:					Any   =       output_from_no_past[:, -3:, random_slice_idx].detach()
										snake_case_				:					List[str]   =       output_from_past[:, :, random_slice_idx].detach()

										self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]				)

										# test that outputs are equal for slice
										self.parent.assertTrue(torch.allclose(lowerCAmelCase__       ,							lowerCAmelCase__       ,							atol=1E-3				)				)

							def 							_A	(		self			:int       ,							lowerCAmelCase__			:Optional[Any]       ,							lowerCAmelCase__			:Dict       ,							lowerCAmelCase__			:int       ,							lowerCAmelCase__			:int       ,							*lowerCAmelCase__			:Optional[Any]       ,							)				->  Optional[int]:


										'''simple docstring'''




										snake_case_				:					Dict   =       BertGenerationDecoder(lowerCAmelCase__				)
										model.to(lowerCAmelCase__				)
										model.eval()
										snake_case_				:					List[str]   =       model(lowerCAmelCase__       ,							attention_mask=lowerCAmelCase__       ,							labels=lowerCAmelCase__				)
										self.parent.assertEqual(result.logits.shape       ,							(self.batch_size, self.seq_length, self.vocab_size)				)






							def 							_A	(		self			:Tuple				)				->  List[Any]:


										'''simple docstring'''




										snake_case_,	snake_case_,	snake_case_,	snake_case_				:					int   =       self.prepare_config_and_inputs()
										snake_case_				:					List[Any]   =       {"input_ids": input_ids, "attention_mask": input_mask}
										return config, inputs_dict







@require_torch
class    A_	(a_						,					a_						,					a_						,					unittest.TestCase							):
							"""simple docstring"""




							a__   		=	(BertGenerationEncoder, BertGenerationDecoder) if is_torch_available() else ()
							a__   		=	(BertGenerationDecoder,) if is_torch_available() else ()
							a__   		=	(
							    {'''feature-extraction''': BertGenerationEncoder, '''text-generation''': BertGenerationDecoder}
							    if is_torch_available()
							    else {}
							)

							def 							_A	(		self			:int				)				->  str:


										'''simple docstring'''




										snake_case_				:					List[str]   =       BertGenerationEncoderTester(self				)
										snake_case_				:					str   =       ConfigTester(self       ,							config_class=lowerCAmelCase__       ,							hidden_size=37				)

							def 							_A	(		self			:Optional[Any]				)				->  Optional[int]:


										'''simple docstring'''




										self.config_tester.run_common_tests()

							def 							_A	(		self			:int				)				->  List[Any]:


										'''simple docstring'''




										snake_case_				:					List[str]   =       self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*lowerCAmelCase__				)

							def 							_A	(		self			:Tuple				)				->  Union[str, Any]:


										'''simple docstring'''




										snake_case_,	snake_case_,	snake_case_,	snake_case_				:					str   =       self.model_tester.prepare_config_and_inputs()
										snake_case_				:					int   =       "bert"
										self.model_tester.create_and_check_model(lowerCAmelCase__       ,							lowerCAmelCase__       ,							lowerCAmelCase__       ,							lowerCAmelCase__				)

							def 							_A	(		self			:List[str]				)				->  Any:


										'''simple docstring'''




										snake_case_				:					List[Any]   =       self.model_tester.prepare_config_and_inputs_for_decoder()
										self.model_tester.create_and_check_model_as_decoder(*lowerCAmelCase__				)

							def 							_A	(		self			:List[Any]				)				->  Dict:


										'''simple docstring'''




										snake_case_				:					int   =       self.model_tester.prepare_config_and_inputs_for_decoder()
										self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCAmelCase__				)

							def 							_A	(		self			:Optional[Any]				)				->  Any:


										'''simple docstring'''




										(
										    (
										    snake_case_
										),	(
										    snake_case_
										),	(
										    snake_case_
										),	(
										    snake_case_
										),	(
										    snake_case_
										),	(
										    snake_case_
										),	
										)				:					List[Any]   =       self.model_tester.prepare_config_and_inputs_for_decoder()

										snake_case_				:					Optional[Any]   =       None

										self.model_tester.create_and_check_model_as_decoder(
										    lowerCAmelCase__       ,							lowerCAmelCase__       ,							lowerCAmelCase__       ,							lowerCAmelCase__       ,							lowerCAmelCase__       ,							lowerCAmelCase__       ,							)

							def 							_A	(		self			:Optional[Any]				)				->  Dict:


										'''simple docstring'''




										snake_case_				:					Optional[int]   =       self.model_tester.prepare_config_and_inputs_for_decoder()
										self.model_tester.create_and_check_for_causal_lm(*lowerCAmelCase__				)






							@slow
							def 							_A	(		self			:Optional[Any]				)				->  str:


										'''simple docstring'''




										snake_case_				:					int   =       BertGenerationEncoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder"				)
										self.assertIsNotNone(lowerCAmelCase__				)







@require_torch
class    A_	(unittest.TestCase							):
							"""simple docstring"""




							@slow
							def 							_A	(		self			:int				)				->  int:


										'''simple docstring'''




										snake_case_				:					Optional[int]   =       BertGenerationEncoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder"				)
										snake_case_				:					str   =       torch.tensor([[101, 7_592, 1_010, 2_026, 3_899, 2_003, 10_140, 102]]				)
										with torch.no_grad():
													snake_case_				:					Optional[Any]   =       model(lowerCAmelCase__				)[0]
										snake_case_				:					Optional[int]   =       torch.Size([1, 8, 1_024]				)
										self.assertEqual(output.shape       ,							lowerCAmelCase__				)
										snake_case_				:					str   =       torch.tensor(
										    [[[0.1_7_7_5, 0.0_0_8_3, -0.0_3_2_1], [1.6_0_0_2, 0.1_2_8_7, 0.3_9_1_2], [2.1_4_7_3, 0.5_7_9_1, 0.6_0_6_6]]]				)
										self.assertTrue(torch.allclose(output[:, :3, :3]       ,							lowerCAmelCase__       ,							atol=1E-4				)				)







@require_torch
class    A_	(unittest.TestCase							):
							"""simple docstring"""




							@slow
							def 							_A	(		self			:Union[str, Any]				)				->  Dict:


										'''simple docstring'''




										snake_case_				:					str   =       BertGenerationDecoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder"				)
										snake_case_				:					Optional[Any]   =       torch.tensor([[101, 7_592, 1_010, 2_026, 3_899, 2_003, 10_140, 102]]				)
										with torch.no_grad():
													snake_case_				:					List[Any]   =       model(lowerCAmelCase__				)[0]
										snake_case_				:					Optional[Any]   =       torch.Size([1, 8, 50_358]				)
										self.assertEqual(output.shape       ,							lowerCAmelCase__				)
										snake_case_				:					Optional[Any]   =       torch.tensor(
										    [[[-0.5_7_8_8, -2.5_9_9_4, -3.7_0_5_4], [0.0_4_3_8, 4.7_9_9_7, 1.8_7_9_5], [1.5_8_6_2, 6.6_4_0_9, 4.4_6_3_8]]]				)
										self.assertTrue(torch.allclose(output[:, :3, :3]       ,							lowerCAmelCase__       ,							atol=1E-4				)				)












'''simple docstring'''




import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment

def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__=()     ,__magic_name__=None     ,__magic_name__="no"     ,__magic_name__="29500"						)->    Optional[int]:



			"""simple docstring"""



			snake_case_				:					str   =       False
			snake_case_				:					int   =       False
			if any(key.startswith("KAGGLE"						) for key in os.environ.keys()						):
						snake_case_				:					Any   =       True
			elif "IPython" in sys.modules:
						snake_case_				:					Union[str, Any]   =       "google.colab" in str(sys.modules["IPython"].get_ipython()						)

			try:
						snake_case_				:					Any   =       PrecisionType(mixed_precision.lower()						)
			except ValueError:
						raise ValueError(
						    F'''Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.'''						)

			if (in_colab or in_kaggle) and (os.environ.get("TPU_NAME"     ,__magic_name__						) is not None):
						# TPU launch
						import torch_xla.distributed.xla_multiprocessing as xmp

						if len(AcceleratorState._shared_state						) > 0:
									raise ValueError(
									    "To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside "
									    "your training function. Restart your notebook and make sure no cells initializes an "
									    "`Accelerator`."						)
						if num_processes is None:
									snake_case_				:					Tuple   =       8

						snake_case_				:					Optional[int]   =       PrepareForLaunch(__magic_name__     ,distributed_type="TPU"						)
						print(F'''Launching a training on {num_processes} TPU cores.'''						)
						xmp.spawn(__magic_name__     ,args=__magic_name__     ,nprocs=__magic_name__     ,start_method="fork"						)
			elif in_colab:
						# No need for a distributed launch otherwise as it's either CPU or one GPU.
						if torch.cuda.is_available():
									print("Launching training on one GPU."						)
						else:
									print("Launching training on one CPU."						)
						function(*__magic_name__						)
			else:
						if num_processes is None:
									raise ValueError(
									    "You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call."						)

						if num_processes > 1:
									# Multi-GPU launch
									from torch.multiprocessing import start_processes
									from torch.multiprocessing.spawn import ProcessRaisedException

									if len(AcceleratorState._shared_state						) > 0:
												raise ValueError(
												    "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
												    "inside your training function. Restart your notebook and make sure no cells initializes an "
												    "`Accelerator`."						)

									if torch.cuda.is_initialized():
												raise ValueError(
												    "To launch a multi-GPU training from your notebook, you need to avoid running any instruction "
												    "using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA "
												    "function."						)

									# torch.distributed will expect a few environment variable to be here. We set the ones common to each
									# process here (the other ones will be set be the launcher).
									with patch_environment(
									    world_size=__magic_name__     ,master_addr="127.0.01"     ,master_port=__magic_name__     ,mixed_precision=__magic_name__						):
												snake_case_				:					Optional[int]   =       PrepareForLaunch(__magic_name__     ,distributed_type="MULTI_GPU"						)
												print(F'''Launching training on {num_processes} GPUs.'''						)
												try:
															start_processes(__magic_name__     ,args=__magic_name__     ,nprocs=__magic_name__     ,start_method="fork"						)
												except ProcessRaisedException as e:
															if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																		raise RuntimeError(
																		    "CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. "
																		    "This likely stems from an outside import causing issues once the `notebook_launcher()` is called. "
																		    "Please review your imports and test them when running the `notebook_launcher()` to identify "
																		    "which one is problematic."						) from e

						else:
									# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
									if is_mps_available():
												snake_case_				:					Any   =       "1"
												print("Launching training on MPS."						)
									elif torch.cuda.is_available():
												print("Launching training on one GPU."						)
									else:
												print("Launching training on CPU."						)
									function(*__magic_name__						)






def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__=()     ,__magic_name__=2						)->    Dict:



			"""simple docstring"""



			from torch.multiprocessing import start_processes

			with tempfile.NamedTemporaryFile() as tmp_file:
						# torch.distributed will expect a few environment variable to be here. We set the ones common to each
						# process here (the other ones will be set be the launcher).
						with patch_environment(
						    world_size=__magic_name__     ,master_addr="127.0.01"     ,master_port="29500"     ,accelerate_mixed_precision="no"     ,accelerate_debug_rdv_file=tmp_file.name     ,accelerate_use_cpu="yes"     ,):
									snake_case_				:					Any   =       PrepareForLaunch(__magic_name__     ,debug=__magic_name__						)
									start_processes(__magic_name__     ,args=__magic_name__     ,nprocs=__magic_name__     ,start_method="fork"						)




