"""simple docstring"""




import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel

class       a_      :




					def __init__(	self						:     Any  ,						snake_case__						:     int  ,						snake_case__						:     Dict=13  ,						snake_case__						:     Union[str, Any]=7  ,						snake_case__						:     int=True  ,						snake_case__						:     Dict=True  ,						snake_case__						:     str=False  ,						snake_case__						:     Optional[int]=True  ,						snake_case__						:     Dict=99  ,						snake_case__						:     Dict=32  ,						snake_case__						:     Tuple=5  ,						snake_case__						:     int=4  ,						snake_case__						:     List[str]=37  ,						snake_case__						:     List[Any]="gelu"  ,						snake_case__						:     str=0.1  ,						snake_case__						:     Tuple=0.1  ,						snake_case__						:     Union[str, Any]=512  ,						snake_case__						:     Tuple=16  ,						snake_case__						:     str=2  ,						snake_case__						:     Optional[Any]=0.02  ,						snake_case__						:     Optional[Any]=3  ,						snake_case__						:     Tuple=4  ,						snake_case__						:     Dict=None  ,						):
							lowerCAmelCase__        =   parent
							lowerCAmelCase__        =   batch_size
							lowerCAmelCase__        =   seq_length
							lowerCAmelCase__        =   is_training
							lowerCAmelCase__        =   use_input_mask
							lowerCAmelCase__        =   use_token_type_ids
							lowerCAmelCase__        =   use_labels
							lowerCAmelCase__        =   vocab_size
							lowerCAmelCase__        =   hidden_size
							lowerCAmelCase__        =   num_hidden_layers
							lowerCAmelCase__        =   num_attention_heads
							lowerCAmelCase__        =   intermediate_size
							lowerCAmelCase__        =   hidden_act
							lowerCAmelCase__        =   hidden_dropout_prob
							lowerCAmelCase__        =   attention_probs_dropout_prob
							lowerCAmelCase__        =   max_position_embeddings
							lowerCAmelCase__        =   type_vocab_size
							lowerCAmelCase__        =   type_sequence_label_size
							lowerCAmelCase__        =   initializer_range
							lowerCAmelCase__        =   num_labels
							lowerCAmelCase__        =   num_choices
							lowerCAmelCase__        =   scope




					def 						_SCREAMING_SNAKE_CASE						(	self						:     str     ):
							lowerCAmelCase__        =   ids_tensor([self.batch_size, self.seq_length]  ,						self.vocab_size     )

							lowerCAmelCase__        =   None
							if self.use_input_mask:
									lowerCAmelCase__        =   random_attention_mask([self.batch_size, self.seq_length]     )

							lowerCAmelCase__        =   None
							if self.use_token_type_ids:
									lowerCAmelCase__        =   ids_tensor([self.batch_size, self.seq_length]  ,						self.type_vocab_size     )

							lowerCAmelCase__        =   None
							lowerCAmelCase__        =   None
							lowerCAmelCase__        =   None
							if self.use_labels:
									lowerCAmelCase__        =   ids_tensor([self.batch_size]  ,						self.type_sequence_label_size     )
									lowerCAmelCase__        =   ids_tensor([self.batch_size, self.seq_length]  ,						self.num_labels     )
									lowerCAmelCase__        =   ids_tensor([self.batch_size]  ,						self.num_choices     )

							lowerCAmelCase__        =   self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




					def 						_SCREAMING_SNAKE_CASE						(	self						:     str     ):
							return OpenLlamaConfig(
							    vocab_size=self.vocab_size  ,						hidden_size=self.hidden_size  ,						num_hidden_layers=self.num_hidden_layers  ,						num_attention_heads=self.num_attention_heads  ,						intermediate_size=self.intermediate_size  ,						hidden_act=self.hidden_act  ,						hidden_dropout_prob=self.hidden_dropout_prob  ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,						max_position_embeddings=self.max_position_embeddings  ,						type_vocab_size=self.type_vocab_size  ,						is_decoder=snake_case__  ,						initializer_range=self.initializer_range  ,						use_stable_embedding=snake_case__  ,						)




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Tuple  ,						snake_case__						:     Dict  ,						snake_case__						:     int  ,						snake_case__						:     Union[str, Any]  ,						snake_case__						:     Tuple  ,						snake_case__						:     List[str]  ,						snake_case__						:     str  ,						snake_case__						:     Dict     ):
							lowerCAmelCase__        =   OpenLlamaModel(config=snake_case__     )
							model.to(snake_case__     )
							model.eval()
							lowerCAmelCase__        =   model(snake_case__  ,						attention_mask=snake_case__     )
							lowerCAmelCase__        =   model(snake_case__     )
							self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.hidden_size)     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Optional[int]  ,						snake_case__						:     Any  ,						snake_case__						:     List[str]  ,						snake_case__						:     Any  ,						snake_case__						:     Optional[Any]  ,						snake_case__						:     Union[str, Any]  ,						snake_case__						:     Union[str, Any]  ,						snake_case__						:     int  ,						snake_case__						:     Any  ,						snake_case__						:     Optional[Any]  ,						):
							lowerCAmelCase__        =   True
							lowerCAmelCase__        =   OpenLlamaModel(snake_case__     )
							model.to(snake_case__     )
							model.eval()
							lowerCAmelCase__        =   model(
							    snake_case__  ,						attention_mask=snake_case__  ,						encoder_hidden_states=snake_case__  ,						encoder_attention_mask=snake_case__  ,						)
							lowerCAmelCase__        =   model(
							    snake_case__  ,						attention_mask=snake_case__  ,						encoder_hidden_states=snake_case__  ,						)
							lowerCAmelCase__        =   model(snake_case__  ,						attention_mask=snake_case__     )
							self.parent.assertEqual(result.last_hidden_state.shape  ,						(self.batch_size, self.seq_length, self.hidden_size)     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     List[Any]  ,						snake_case__						:     Tuple  ,						snake_case__						:     Optional[int]  ,						snake_case__						:     int  ,						snake_case__						:     Dict  ,						snake_case__						:     Union[str, Any]  ,						snake_case__						:     Optional[int]  ,						snake_case__						:     Any  ,						snake_case__						:     List[Any]  ,						snake_case__						:     Dict  ,						):
							lowerCAmelCase__        =   OpenLlamaForCausalLM(config=snake_case__     )
							model.to(snake_case__     )
							model.eval()
							lowerCAmelCase__        =   model(snake_case__  ,						attention_mask=snake_case__  ,						labels=snake_case__     )
							self.parent.assertEqual(result.logits.shape  ,						(self.batch_size, self.seq_length, self.vocab_size)     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     List[Any]  ,						snake_case__						:     Optional[Any]  ,						snake_case__						:     str  ,						snake_case__						:     int  ,						snake_case__						:     List[Any]  ,						snake_case__						:     str  ,						snake_case__						:     Optional[int]  ,						snake_case__						:     Dict  ,						snake_case__						:     List[str]  ,						snake_case__						:     Dict  ,						):
							lowerCAmelCase__        =   True
							lowerCAmelCase__        =   True
							lowerCAmelCase__        =   OpenLlamaForCausalLM(config=snake_case__     )
							model.to(snake_case__     )
							model.eval()

							# first forward pass
							lowerCAmelCase__        =   model(
							    snake_case__  ,						attention_mask=snake_case__  ,						encoder_hidden_states=snake_case__  ,						encoder_attention_mask=snake_case__  ,						use_cache=snake_case__  ,						)
							lowerCAmelCase__        =   outputs.past_key_values

							# create hypothetical multiple next token and extent to next_input_ids
							lowerCAmelCase__        =   ids_tensor((self.batch_size, 3)  ,						config.vocab_size     )
							lowerCAmelCase__        =   ids_tensor((self.batch_size, 3)  ,						vocab_size=2     )

							# append to next input_ids and
							lowerCAmelCase__        =   torch.cat([input_ids, next_tokens]  ,						dim=-1     )
							lowerCAmelCase__        =   torch.cat([input_mask, next_mask]  ,						dim=-1     )

							lowerCAmelCase__        =   model(
							    snake_case__  ,						attention_mask=snake_case__  ,						encoder_hidden_states=snake_case__  ,						encoder_attention_mask=snake_case__  ,						output_hidden_states=snake_case__  ,						)['''hidden_states'''][0]
							lowerCAmelCase__        =   model(
							    snake_case__  ,						attention_mask=snake_case__  ,						encoder_hidden_states=snake_case__  ,						encoder_attention_mask=snake_case__  ,						past_key_values=snake_case__  ,						output_hidden_states=snake_case__  ,						)['''hidden_states'''][0]

							# select random slice
							lowerCAmelCase__        =   ids_tensor((1,)  ,						output_from_past.shape[-1]     ).item()
							lowerCAmelCase__        =   output_from_no_past[:, -3:, random_slice_idx].detach()
							lowerCAmelCase__        =   output_from_past[:, :, random_slice_idx].detach()

							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(snake_case__  ,						snake_case__  ,						atol=1E-3     )     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]     ):
							lowerCAmelCase__        =   self.prepare_config_and_inputs()
							(
							    lowerCAmelCase__	
							)        =   config_and_inputs
							lowerCAmelCase__        =   {'''input_ids''': input_ids, '''attention_mask''': input_mask}
							return config, inputs_dict

@require_torch
class       a_      (     UpperCamelCase_ ,		UpperCamelCase_ ,		UpperCamelCase_ ,		unittest.TestCase			):
					UpperCamelCase_ :	Union[str, Any]			=    (
					    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
					)
					UpperCamelCase_ :	List[str]			=    (OpenLlamaForCausalLM,) if is_torch_available() else ()
					UpperCamelCase_ :	List[str]			=    (
					    {
					        "feature-extraction": OpenLlamaModel,
					        "text-classification": OpenLlamaForSequenceClassification,
					        "text-generation": OpenLlamaForCausalLM,
					        "zero-shot": OpenLlamaForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					UpperCamelCase_ :	Optional[Any]			=    False
					UpperCamelCase_ :	Optional[int]			=    False




					def 						_SCREAMING_SNAKE_CASE						(	self						:     int     ):
							lowerCAmelCase__        =   OpenLlamaModelTester(self     )
							lowerCAmelCase__        =   ConfigTester(self  ,						config_class=snake_case__  ,						hidden_size=37     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Tuple     ):
							self.config_tester.run_common_tests()




					def 						_SCREAMING_SNAKE_CASE						(	self						:     List[Any]     ):
							lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*snake_case__     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     int     ):
							lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
									lowerCAmelCase__        =   type
									self.model_tester.create_and_check_model(*snake_case__     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     str     ):
							lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs_for_common()
							lowerCAmelCase__        =   3
							lowerCAmelCase__        =   input_dict['''input_ids''']
							lowerCAmelCase__        =   input_ids.ne(1     ).to(snake_case__     )
							lowerCAmelCase__        =   ids_tensor([self.model_tester.batch_size]  ,						self.model_tester.type_sequence_label_size     )
							lowerCAmelCase__        =   OpenLlamaForSequenceClassification(snake_case__     )
							model.to(snake_case__     )
							model.eval()
							lowerCAmelCase__        =   model(snake_case__  ,						attention_mask=snake_case__  ,						labels=snake_case__     )
							self.assertEqual(result.logits.shape  ,						(self.model_tester.batch_size, self.model_tester.num_labels)     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Union[str, Any]     ):
							lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs_for_common()
							lowerCAmelCase__        =   3
							lowerCAmelCase__        =   '''single_label_classification'''
							lowerCAmelCase__        =   input_dict['''input_ids''']
							lowerCAmelCase__        =   input_ids.ne(1     ).to(snake_case__     )
							lowerCAmelCase__        =   ids_tensor([self.model_tester.batch_size]  ,						self.model_tester.type_sequence_label_size     )
							lowerCAmelCase__        =   OpenLlamaForSequenceClassification(snake_case__     )
							model.to(snake_case__     )
							model.eval()
							lowerCAmelCase__        =   model(snake_case__  ,						attention_mask=snake_case__  ,						labels=snake_case__     )
							self.assertEqual(result.logits.shape  ,						(self.model_tester.batch_size, self.model_tester.num_labels)     )




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Optional[int]     ):
							lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs_for_common()
							lowerCAmelCase__        =   3
							lowerCAmelCase__        =   '''multi_label_classification'''
							lowerCAmelCase__        =   input_dict['''input_ids''']
							lowerCAmelCase__        =   input_ids.ne(1     ).to(snake_case__     )
							lowerCAmelCase__        =   ids_tensor(
							    [self.model_tester.batch_size, config.num_labels]  ,						self.model_tester.type_sequence_label_size     ).to(torch.float     )
							lowerCAmelCase__        =   OpenLlamaForSequenceClassification(snake_case__     )
							model.to(snake_case__     )
							model.eval()
							lowerCAmelCase__        =   model(snake_case__  ,						attention_mask=snake_case__  ,						labels=snake_case__     )
							self.assertEqual(result.logits.shape  ,						(self.model_tester.batch_size, self.model_tester.num_labels)     )




					@unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test"""     )
					def 						_SCREAMING_SNAKE_CASE						(	self						:     Optional[int]     ):
							pass




					@parameterized.expand([("""linear""",), ("""dynamic""",)]     )
					def 						_SCREAMING_SNAKE_CASE						(	self						:     Optional[int]  ,						snake_case__						:     Any     ):
							lowerCAmelCase__        =   self.model_tester.prepare_config_and_inputs_for_common()
							lowerCAmelCase__        =   ids_tensor([1, 10]  ,						config.vocab_size     )
							lowerCAmelCase__        =   ids_tensor([1, int(config.max_position_embeddings * 1.5     )]  ,						config.vocab_size     )

							set_seed(42     )  # Fixed seed at init time so the two models get the same random weights
							lowerCAmelCase__        =   OpenLlamaModel(snake_case__     )
							original_model.to(snake_case__     )
							original_model.eval()
							lowerCAmelCase__        =   original_model(snake_case__     ).last_hidden_state
							lowerCAmelCase__        =   original_model(snake_case__     ).last_hidden_state

							set_seed(42     )  # Fixed seed at init time so the two models get the same random weights
							lowerCAmelCase__        =   {'''type''': scaling_type, '''factor''': 10.0}
							lowerCAmelCase__        =   OpenLlamaModel(snake_case__     )
							scaled_model.to(snake_case__     )
							scaled_model.eval()
							lowerCAmelCase__        =   scaled_model(snake_case__     ).last_hidden_state
							lowerCAmelCase__        =   scaled_model(snake_case__     ).last_hidden_state

							# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
							# maximum sequence length, so the outputs for the short input should match.
							if scaling_type == "dynamic":
									self.assertTrue(torch.allclose(snake_case__  ,						snake_case__  ,						atol=1E-5     )     )
							else:
									self.assertFalse(torch.allclose(snake_case__  ,						snake_case__  ,						atol=1E-5     )     )

							# The output should be different for long inputs
							self.assertFalse(torch.allclose(snake_case__  ,						snake_case__  ,						atol=1E-5     )     )





"""simple docstring"""




import numpy as np
import torch
from torch.utils.data import DataLoader

from accelerate.utils.dataclasses import DistributedType

class       a_      :




					def __init__(	self						:     Optional[int]  ,						snake_case__						:     List[Any]=2  ,						snake_case__						:     Any=3  ,						snake_case__						:     Union[str, Any]=64  ,						snake_case__						:     Any=None     ):
							lowerCAmelCase__        =   np.random.default_rng(snake_case__     )
							lowerCAmelCase__        =   length
							lowerCAmelCase__        =   rng.normal(size=(length,)     ).astype(np.floataa     )
							lowerCAmelCase__        =   a * self.x + b + rng.normal(scale=0.1  ,						size=(length,)     ).astype(np.floataa     )




					def __len__(	self						:     Optional[Any]     ):
							return self.length




					def __getitem__(	self						:     List[str]  ,						snake_case__						:     Optional[int]     ):
							return {"x": self.x[i], "y": self.y[i]}

class       a_      (     torch.nn.Module			):




					def __init__(	self						:     List[str]  ,						snake_case__						:     str=0  ,						snake_case__						:     Dict=0  ,						snake_case__						:     Any=False     ):
							super().__init__()
							lowerCAmelCase__        =   torch.nn.Parameter(torch.tensor([2, 3]     ).float()     )
							lowerCAmelCase__        =   torch.nn.Parameter(torch.tensor([2, 3]     ).float()     )
							lowerCAmelCase__        =   True




					def 						_SCREAMING_SNAKE_CASE						(	self						:     int  ,						snake_case__						:     Any=None     ):
							if self.first_batch:
									print(F"""Model dtype: {self.a.dtype}, {self.b.dtype}. Input dtype: {x.dtype}"""     )
									lowerCAmelCase__        =   False
							return x * self.a[0] + self.b[0]

class       a_      (     torch.nn.Module			):




					def __init__(	self						:     Any  ,						snake_case__						:     Union[str, Any]=0  ,						snake_case__						:     Union[str, Any]=0  ,						snake_case__						:     List[Any]=False     ):
							super().__init__()
							lowerCAmelCase__        =   torch.nn.Parameter(torch.tensor(snake_case__     ).float()     )
							lowerCAmelCase__        =   torch.nn.Parameter(torch.tensor(snake_case__     ).float()     )
							lowerCAmelCase__        =   True




					def 						_SCREAMING_SNAKE_CASE						(	self						:     Tuple  ,						snake_case__						:     Optional[Any]=None     ):
							if self.first_batch:
									print(F"""Model dtype: {self.a.dtype}, {self.b.dtype}. Input dtype: {x.dtype}"""     )
									lowerCAmelCase__        =   False
							return x * self.a + self.b







def 							_UpperCAmelCase		(       lowerCamelCase__			,  lowerCamelCase__ = 16     ):





		"""simple docstring"""




		from datasets import load_dataset
		from transformers import AutoTokenizer

		lowerCAmelCase__        =   AutoTokenizer.from_pretrained("""bert-base-cased"""     )
		lowerCAmelCase__        =   {"""train""": """tests/test_samples/MRPC/train.csv""", """validation""": """tests/test_samples/MRPC/dev.csv"""}
		lowerCAmelCase__        =   load_dataset("""csv"""			,  data_files=lowerCamelCase__     )
		lowerCAmelCase__        =   datasets["""train"""].unique("""label"""     )

		lowerCAmelCase__        =   {v: i for i, v in enumerate(lowerCamelCase__     )}

		def tokenize_function(lowerCamelCase__     ):
				# max_length=None => use the model max length (it's actually the default)
				lowerCAmelCase__        =   tokenizer(
				    examples["""sentence1"""]			,  examples["""sentence2"""]			,  truncation=lowerCamelCase__			,  max_length=lowerCamelCase__			,  padding="""max_length"""     )
				if "label" in examples:
						lowerCAmelCase__        =   [label_to_id[l] for l in examples["""label"""]]
				return outputs

		# Apply the method we just defined to all the examples in all the splits of the dataset
		lowerCAmelCase__        =   datasets.map(
		    lowerCamelCase__			,  batched=lowerCamelCase__			,  remove_columns=["""sentence1""", """sentence2""", """label"""]			,  )

		def collate_fn(lowerCamelCase__     ):
				# On TPU it's best to pad everything to the same length or training will be very slow.
				if accelerator.distributed_type == DistributedType.TPU:
						return tokenizer.pad(lowerCamelCase__			,  padding="""max_length"""			,  max_length=128			,  return_tensors="""pt"""     )
				return tokenizer.pad(lowerCamelCase__			,  padding="""longest"""			,  return_tensors="""pt"""     )

		# Instantiate dataloaders.
		lowerCAmelCase__        =   DataLoader(tokenized_datasets["""train"""]			,  shuffle=lowerCamelCase__			,  collate_fn=lowerCamelCase__			,  batch_size=2     )
		lowerCAmelCase__        =   DataLoader(tokenized_datasets["""validation"""]			,  shuffle=lowerCamelCase__			,  collate_fn=lowerCamelCase__			,  batch_size=1     )

		return train_dataloader, eval_dataloader




