


import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.local_sgd import LocalSGD


########################################################################
# This is a fully working simple example to use Accelerate
# with LocalSGD, which is a method to synchronize model
# parameters every K batches. It is different, but complementary
# to gradient accumulation.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


a					  =     1_6
a					  =     3_2


def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__ = 1_6  ):
					lowercase_  							=			AutoTokenizer.from_pretrained("""bert-base-cased"""  )
					lowercase_  							=			load_dataset("""glue"""						,							"""mrpc"""  )

					def tokenize_function(UpperCAmelCase__  ):
										# max_length=None => use the model max length (it's actually the default)
										lowercase_  							=			tokenizer(examples["""sentence1"""]						,							examples["""sentence2"""]						,							truncation=UpperCAmelCase__						,							max_length=UpperCAmelCase__  )
										return outputs

					# Apply the method we just defined to all the examples in all the splits of the dataset
					# starting with the main process first:
					with accelerator.main_process_first():
										lowercase_  							=			datasets.map(
										    UpperCAmelCase__						,							batched=UpperCAmelCase__						,							remove_columns=["""idx""", """sentence1""", """sentence2"""]						,							)

					# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
					# transformers library
					lowercase_  							=			tokenized_datasets.rename_column("""label"""						,							"""labels"""  )

					def collate_fn(UpperCAmelCase__  ):
										# On TPU it's best to pad everything to the same length or training will be very slow.
										lowercase_  							=			1_2_8 if accelerator.distributed_type == DistributedType.TPU else None
										# When using mixed precision we want round multiples of 8/16
										if accelerator.mixed_precision == "fp8":
															lowercase_  							=			1_6
										elif accelerator.mixed_precision != "no":
															lowercase_  							=			8
										else:
															lowercase_  							=			None

										return tokenizer.pad(
										    UpperCAmelCase__						,							padding="""longest"""						,							max_length=UpperCAmelCase__						,							pad_to_multiple_of=UpperCAmelCase__						,							return_tensors="""pt"""						,							)

					# Instantiate dataloaders.
					lowercase_  							=			DataLoader(
					    tokenized_datasets["""train"""]						,							shuffle=UpperCAmelCase__						,							collate_fn=UpperCAmelCase__						,							batch_size=UpperCAmelCase__  )
					lowercase_  							=			DataLoader(
					    tokenized_datasets["""validation"""]						,							shuffle=UpperCAmelCase__						,							collate_fn=UpperCAmelCase__						,							batch_size=UpperCAmelCase__  )

					return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('TESTING_MOCKED_DATALOADERS', None) == "1":
							from accelerate.test_utils.training import mocked_dataloaders

							a					  =     mocked_dataloaders  # noqa: F811


def 			UpperCAmelCase_				(      UpperCAmelCase__						,							UpperCAmelCase__  ):
					# For testing only
					if os.environ.get("""TESTING_MOCKED_DATALOADERS"""						,							UpperCAmelCase__  ) == "1":
										lowercase_  							=			2
					# New Code #
					lowercase_  							=			int(args.gradient_accumulation_steps  )
					lowercase_  							=			int(args.local_sgd_steps  )
					# Initialize accelerator
					lowercase_  							=			Accelerator(
					    cpu=args.cpu						,							mixed_precision=args.mixed_precision						,							gradient_accumulation_steps=UpperCAmelCase__  )
					if accelerator.distributed_type not in [DistributedType.NO, DistributedType.MULTI_CPU, DistributedType.MULTI_GPU]:
										raise NotImplementedError("""LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)"""  )
					# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
					lowercase_  							=			config["""lr"""]
					lowercase_  							=			int(config["""num_epochs"""]  )
					lowercase_  							=			int(config["""seed"""]  )
					lowercase_  							=			int(config["""batch_size"""]  )

					lowercase_  							=			evaluate.load("""glue"""						,							"""mrpc"""  )

					set_seed(UpperCAmelCase__  )
					lowercase_							,					lowercase_  							=			get_dataloaders(UpperCAmelCase__						,							UpperCAmelCase__  )
					# Instantiate the model (we build the model here so that the seed also control new weights initialization)
					lowercase_  							=			AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""						,							return_dict=UpperCAmelCase__  )

					# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
					# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
					# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
					lowercase_  							=			model.to(accelerator.device  )

					# Instantiate optimizer
					lowercase_  							=			AdamW(params=model.parameters()						,							lr=UpperCAmelCase__  )

					# Instantiate scheduler
					lowercase_  							=			get_linear_schedule_with_warmup(
					    optimizer=UpperCAmelCase__						,							num_warmup_steps=1_0_0						,							num_training_steps=(len(UpperCAmelCase__  ) * num_epochs)						,							)

					# Prepare everything
					# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
					# prepare method.
					lowercase_							,					lowercase_							,					lowercase_							,					lowercase_							,					lowercase_  							=			accelerator.prepare(
					    UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__						,							UpperCAmelCase__  )

					# Now we train the model
					for epoch in range(UpperCAmelCase__  ):
										model.train()
										with LocalSGD(
										    accelerator=UpperCAmelCase__						,							model=UpperCAmelCase__						,							local_sgd_steps=UpperCAmelCase__						,							enabled=local_sgd_steps is not None  ) as local_sgd:
															for step, batch in enumerate(UpperCAmelCase__  ):
																				# We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device  )
																				# New code #
																				# We use the new `accumulate` context manager to perform gradient accumulation
																				# We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
																				with accelerator.accumulate(UpperCAmelCase__  ):
																									lowercase_  							=			model(**UpperCAmelCase__  )
																									lowercase_  							=			output.loss
																									accelerator.backward(UpperCAmelCase__  )
																									optimizer.step()
																									lr_scheduler.step()
																									optimizer.zero_grad()
																									# LocalSGD-specific line
																									local_sgd.step()

										model.eval()
										for step, batch in enumerate(UpperCAmelCase__  ):
															# We could avoid this line since we set the accelerator with `device_placement=True`.
															batch.to(accelerator.device  )
															with torch.no_grad():
																				lowercase_  							=			model(**UpperCAmelCase__  )
															lowercase_  							=			outputs.logits.argmax(dim=-1  )
															lowercase_							,					lowercase_  							=			accelerator.gather_for_metrics((predictions, batch["""labels"""])  )
															metric.add_batch(
															    predictions=UpperCAmelCase__						,							references=UpperCAmelCase__						,							)

										lowercase_  							=			metric.compute()
										# Use accelerator.print to print only on the main process.
										accelerator.print(F'''epoch {epoch}:'''						,							UpperCAmelCase__  )


def 			UpperCAmelCase_				(      ):
					lowercase_  							=			argparse.ArgumentParser(description="""Simple example of training script."""  )
					parser.add_argument(
					    """--mixed_precision"""						,							type=UpperCAmelCase__						,							default=UpperCAmelCase__						,							choices=["""no""", """fp16""", """bf16""", """fp8"""]						,							help="""Whether to use mixed precision. Choose"""
					    """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
					    """and an Nvidia Ampere GPU."""						,							)
					# New Code #
					parser.add_argument(
					    """--gradient_accumulation_steps"""						,							type=UpperCAmelCase__						,							default=1						,							help="""The number of minibatches to be ran before gradients are accumulated."""						,							)
					parser.add_argument(
					    """--local_sgd_steps"""						,							type=UpperCAmelCase__						,							default=8						,							help="""Number of local SGD steps or None to disable local SGD"""  )
					parser.add_argument("""--cpu"""						,							action="""store_true"""						,							help="""If passed, will train on the CPU."""  )
					lowercase_  							=			parser.parse_args()
					lowercase_  							=			{"""lr""": 2e-5, """num_epochs""": 3, """seed""": 4_2, """batch_size""": 1_6}
					training_function(UpperCAmelCase__						,							UpperCAmelCase__  )


if __name__ == "__main__":
							main()






import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel







class        UpperCamelCase__   :

					def __init__(     self       :   List[str]							,					UpperCamelCase__       :   Tuple							,					UpperCamelCase__       :   List[str]=13							,					UpperCamelCase__       :   str=7							,					UpperCamelCase__       :   List[str]=True							,					UpperCamelCase__       :   List[str]=True							,					UpperCamelCase__       :   str=False							,					UpperCamelCase__       :   str=True							,					UpperCamelCase__       :   Union[str, Any]=99							,					UpperCamelCase__       :   Dict=32							,					UpperCamelCase__       :   Union[str, Any]=5							,					UpperCamelCase__       :   Tuple=4							,					UpperCamelCase__       :   Dict=37							,					UpperCamelCase__       :   Any="gelu"							,					UpperCamelCase__       :   Dict=0.1							,					UpperCamelCase__       :   Tuple=0.1							,					UpperCamelCase__       :   Tuple=512							,					UpperCamelCase__       :   Union[str, Any]=16							,					UpperCamelCase__       :   List[str]=2							,					UpperCamelCase__       :   int=0.02							,					UpperCamelCase__       :   int=3							,					UpperCamelCase__       :   Optional[Any]=4							,					UpperCamelCase__       :   int=None							,					):






										'''simple docstring'''






										lowercase_  							=			parent
										lowercase_  							=			batch_size
										lowercase_  							=			seq_length
										lowercase_  							=			is_training
										lowercase_  							=			use_input_mask
										lowercase_  							=			use_token_type_ids
										lowercase_  							=			use_labels
										lowercase_  							=			vocab_size
										lowercase_  							=			hidden_size
										lowercase_  							=			num_hidden_layers
										lowercase_  							=			num_attention_heads
										lowercase_  							=			intermediate_size
										lowercase_  							=			hidden_act
										lowercase_  							=			hidden_dropout_prob
										lowercase_  							=			attention_probs_dropout_prob
										lowercase_  							=			max_position_embeddings
										lowercase_  							=			type_vocab_size
										lowercase_  							=			type_sequence_label_size
										lowercase_  							=			initializer_range
										lowercase_  							=			num_labels
										lowercase_  							=			num_choices
										lowercase_  							=			scope

					def   UpperCAmelCase__       (     self       :   Optional[Any]       ):






										'''simple docstring'''






										lowercase_  							=			ids_tensor([self.batch_size, self.seq_length]							,					self.vocab_size       )

										lowercase_  							=			None
										if self.use_input_mask:
															lowercase_  							=			random_attention_mask([self.batch_size, self.seq_length]       )

										lowercase_  							=			None
										if self.use_token_type_ids:
															lowercase_  							=			ids_tensor([self.batch_size, self.seq_length]							,					self.type_vocab_size       )

										lowercase_  							=			None
										lowercase_  							=			None
										lowercase_  							=			None
										if self.use_labels:
															lowercase_  							=			ids_tensor([self.batch_size]							,					self.type_sequence_label_size       )
															lowercase_  							=			ids_tensor([self.batch_size, self.seq_length]							,					self.num_labels       )
															lowercase_  							=			ids_tensor([self.batch_size]							,					self.num_choices       )

										lowercase_  							=			self.get_config()

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										return OpenLlamaConfig(
										    vocab_size=self.vocab_size							,					hidden_size=self.hidden_size							,					num_hidden_layers=self.num_hidden_layers							,					num_attention_heads=self.num_attention_heads							,					intermediate_size=self.intermediate_size							,					hidden_act=self.hidden_act							,					hidden_dropout_prob=self.hidden_dropout_prob							,					attention_probs_dropout_prob=self.attention_probs_dropout_prob							,					max_position_embeddings=self.max_position_embeddings							,					type_vocab_size=self.type_vocab_size							,					is_decoder=UpperCamelCase__							,					initializer_range=self.initializer_range							,					use_stable_embedding=UpperCamelCase__							,					)

					def   UpperCAmelCase__       (     self       :   str							,					UpperCamelCase__       :   Optional[Any]							,					UpperCamelCase__       :   Tuple							,					UpperCamelCase__       :   Dict							,					UpperCamelCase__       :   List[str]							,					UpperCamelCase__       :   Dict							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   Optional[Any]       ):






										'''simple docstring'''






										lowercase_  							=			OpenLlamaModel(config=UpperCamelCase__       )
										model.to(UpperCamelCase__       )
										model.eval()
										lowercase_  							=			model(UpperCamelCase__							,					attention_mask=UpperCamelCase__       )
										lowercase_  							=			model(UpperCamelCase__       )
										self.parent.assertEqual(result.last_hidden_state.shape							,					(self.batch_size, self.seq_length, self.hidden_size)       )

					def   UpperCAmelCase__       (     self       :   Union[str, Any]							,					UpperCamelCase__       :   Tuple							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   Tuple							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   Optional[Any]							,					UpperCamelCase__       :   Optional[Any]							,					):






										'''simple docstring'''






										lowercase_  							=			True
										lowercase_  							=			OpenLlamaModel(UpperCamelCase__       )
										model.to(UpperCamelCase__       )
										model.eval()
										lowercase_  							=			model(
										    UpperCamelCase__							,					attention_mask=UpperCamelCase__							,					encoder_hidden_states=UpperCamelCase__							,					encoder_attention_mask=UpperCamelCase__							,					)
										lowercase_  							=			model(
										    UpperCamelCase__							,					attention_mask=UpperCamelCase__							,					encoder_hidden_states=UpperCamelCase__							,					)
										lowercase_  							=			model(UpperCamelCase__							,					attention_mask=UpperCamelCase__       )
										self.parent.assertEqual(result.last_hidden_state.shape							,					(self.batch_size, self.seq_length, self.hidden_size)       )

					def   UpperCAmelCase__       (     self       :   Any							,					UpperCamelCase__       :   List[str]							,					UpperCamelCase__       :   List[Any]							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   Any							,					UpperCamelCase__       :   Union[str, Any]							,					UpperCamelCase__       :   Tuple							,					UpperCamelCase__       :   Any							,					UpperCamelCase__       :   List[Any]							,					):






										'''simple docstring'''






										lowercase_  							=			OpenLlamaForCausalLM(config=UpperCamelCase__       )
										model.to(UpperCamelCase__       )
										model.eval()
										lowercase_  							=			model(UpperCamelCase__							,					attention_mask=UpperCamelCase__							,					labels=UpperCamelCase__       )
										self.parent.assertEqual(result.logits.shape							,					(self.batch_size, self.seq_length, self.vocab_size)       )

					def   UpperCAmelCase__       (     self       :   int							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   Tuple							,					UpperCamelCase__       :   Optional[Any]							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   List[str]							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   Tuple							,					):






										'''simple docstring'''






										lowercase_  							=			True
										lowercase_  							=			True
										lowercase_  							=			OpenLlamaForCausalLM(config=UpperCamelCase__       )
										model.to(UpperCamelCase__       )
										model.eval()

										# first forward pass
										lowercase_  							=			model(
										    UpperCamelCase__							,					attention_mask=UpperCamelCase__							,					encoder_hidden_states=UpperCamelCase__							,					encoder_attention_mask=UpperCamelCase__							,					use_cache=UpperCamelCase__							,					)
										lowercase_  							=			outputs.past_key_values

										# create hypothetical multiple next token and extent to next_input_ids
										lowercase_  							=			ids_tensor((self.batch_size, 3)							,					config.vocab_size       )
										lowercase_  							=			ids_tensor((self.batch_size, 3)							,					vocab_size=2       )

										# append to next input_ids and
										lowercase_  							=			torch.cat([input_ids, next_tokens]							,					dim=-1       )
										lowercase_  							=			torch.cat([input_mask, next_mask]							,					dim=-1       )

										lowercase_  							=			model(
										    UpperCamelCase__							,					attention_mask=UpperCamelCase__							,					encoder_hidden_states=UpperCamelCase__							,					encoder_attention_mask=UpperCamelCase__							,					output_hidden_states=UpperCamelCase__							,					)["""hidden_states"""][0]
										lowercase_  							=			model(
										    UpperCamelCase__							,					attention_mask=UpperCamelCase__							,					encoder_hidden_states=UpperCamelCase__							,					encoder_attention_mask=UpperCamelCase__							,					past_key_values=UpperCamelCase__							,					output_hidden_states=UpperCamelCase__							,					)["""hidden_states"""][0]

										# select random slice
										lowercase_  							=			ids_tensor((1,)							,					output_from_past.shape[-1]       ).item()
										lowercase_  							=			output_from_no_past[:, -3:, random_slice_idx].detach()
										lowercase_  							=			output_from_past[:, :, random_slice_idx].detach()

										self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )

										# test that outputs are equal for slice
										self.parent.assertTrue(torch.allclose(UpperCamelCase__							,					UpperCamelCase__							,					atol=1e-3       )       )







					def   UpperCAmelCase__       (     self       :   Optional[Any]       ):






										'''simple docstring'''






										lowercase_  							=			self.prepare_config_and_inputs()
										(
										    (
										    lowercase_
										)							,					(
										    lowercase_
										)							,					(
										    lowercase_
										)							,					(
										    lowercase_
										)							,					(
										    lowercase_
										)							,					(
										    lowercase_
										)							,					(
										    lowercase_
										)							,					
										)  							=			config_and_inputs
										lowercase_  							=			{"""input_ids""": input_ids, """attention_mask""": input_mask}
										return config, inputs_dict







@require_torch
class        UpperCamelCase__   (	__magic_name__							,				__magic_name__							,				__magic_name__							,				unittest.TestCase   ):
					__SCREAMING_SNAKE_CASE       :  Union[str, Any]						=   (
					    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
					)
					__SCREAMING_SNAKE_CASE       :  List[str]						=   (OpenLlamaForCausalLM,) if is_torch_available() else ()
					__SCREAMING_SNAKE_CASE       :  List[Any]						=   (
					    {
					        'feature-extraction': OpenLlamaModel,
					        'text-classification': OpenLlamaForSequenceClassification,
					        'text-generation': OpenLlamaForCausalLM,
					        'zero-shot': OpenLlamaForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					__SCREAMING_SNAKE_CASE       :  Tuple						=   False
					__SCREAMING_SNAKE_CASE       :  Optional[int]						=   False

					def   UpperCAmelCase__       (     self       :   List[Any]       ):






										'''simple docstring'''






										lowercase_  							=			OpenLlamaModelTester(self       )
										lowercase_  							=			ConfigTester(self							,					config_class=UpperCamelCase__							,					hidden_size=37       )

					def   UpperCAmelCase__       (     self       :   Optional[Any]       ):






										'''simple docstring'''






										self.config_tester.run_common_tests()

					def   UpperCAmelCase__       (     self       :   str       ):






										'''simple docstring'''






										lowercase_  							=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										lowercase_  							=			self.model_tester.prepare_config_and_inputs()
										for type in ["absolute", "relative_key", "relative_key_query"]:
															lowercase_  							=			type
															self.model_tester.create_and_check_model(*UpperCamelCase__       )

					def   UpperCAmelCase__       (     self       :   List[Any]       ):






										'''simple docstring'''






										lowercase_							,					lowercase_  							=			self.model_tester.prepare_config_and_inputs_for_common()
										lowercase_  							=			3
										lowercase_  							=			input_dict["""input_ids"""]
										lowercase_  							=			input_ids.ne(1       ).to(UpperCamelCase__       )
										lowercase_  							=			ids_tensor([self.model_tester.batch_size]							,					self.model_tester.type_sequence_label_size       )
										lowercase_  							=			OpenLlamaForSequenceClassification(UpperCamelCase__       )
										model.to(UpperCamelCase__       )
										model.eval()
										lowercase_  							=			model(UpperCamelCase__							,					attention_mask=UpperCamelCase__							,					labels=UpperCamelCase__       )
										self.assertEqual(result.logits.shape							,					(self.model_tester.batch_size, self.model_tester.num_labels)       )

					def   UpperCAmelCase__       (     self       :   Union[str, Any]       ):






										'''simple docstring'''






										lowercase_							,					lowercase_  							=			self.model_tester.prepare_config_and_inputs_for_common()
										lowercase_  							=			3
										lowercase_  							=			"""single_label_classification"""
										lowercase_  							=			input_dict["""input_ids"""]
										lowercase_  							=			input_ids.ne(1       ).to(UpperCamelCase__       )
										lowercase_  							=			ids_tensor([self.model_tester.batch_size]							,					self.model_tester.type_sequence_label_size       )
										lowercase_  							=			OpenLlamaForSequenceClassification(UpperCamelCase__       )
										model.to(UpperCamelCase__       )
										model.eval()
										lowercase_  							=			model(UpperCamelCase__							,					attention_mask=UpperCamelCase__							,					labels=UpperCamelCase__       )
										self.assertEqual(result.logits.shape							,					(self.model_tester.batch_size, self.model_tester.num_labels)       )

					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										lowercase_							,					lowercase_  							=			self.model_tester.prepare_config_and_inputs_for_common()
										lowercase_  							=			3
										lowercase_  							=			"""multi_label_classification"""
										lowercase_  							=			input_dict["""input_ids"""]
										lowercase_  							=			input_ids.ne(1       ).to(UpperCamelCase__       )
										lowercase_  							=			ids_tensor(
										    [self.model_tester.batch_size, config.num_labels]							,					self.model_tester.type_sequence_label_size       ).to(torch.float       )
										lowercase_  							=			OpenLlamaForSequenceClassification(UpperCamelCase__       )
										model.to(UpperCamelCase__       )
										model.eval()
										lowercase_  							=			model(UpperCamelCase__							,					attention_mask=UpperCamelCase__							,					labels=UpperCamelCase__       )
										self.assertEqual(result.logits.shape							,					(self.model_tester.batch_size, self.model_tester.num_labels)       )

					@unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test"""       )
					def   UpperCAmelCase__       (     self       :   Dict       ):






										'''simple docstring'''






										pass







					@parameterized.expand([("""linear""",), ("""dynamic""",)]       )
					def   UpperCAmelCase__       (     self       :   Tuple							,					UpperCamelCase__       :   Union[str, Any]       ):






										'''simple docstring'''






										lowercase_							,					lowercase_  							=			self.model_tester.prepare_config_and_inputs_for_common()
										lowercase_  							=			ids_tensor([1, 10]							,					config.vocab_size       )
										lowercase_  							=			ids_tensor([1, int(config.max_position_embeddings * 1.5       )]							,					config.vocab_size       )

										set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
										lowercase_  							=			OpenLlamaModel(UpperCamelCase__       )
										original_model.to(UpperCamelCase__       )
										original_model.eval()
										lowercase_  							=			original_model(UpperCamelCase__       ).last_hidden_state
										lowercase_  							=			original_model(UpperCamelCase__       ).last_hidden_state

										set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
										lowercase_  							=			{"""type""": scaling_type, """factor""": 10.0}
										lowercase_  							=			OpenLlamaModel(UpperCamelCase__       )
										scaled_model.to(UpperCamelCase__       )
										scaled_model.eval()
										lowercase_  							=			scaled_model(UpperCamelCase__       ).last_hidden_state
										lowercase_  							=			scaled_model(UpperCamelCase__       ).last_hidden_state

										# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
										# maximum sequence length, so the outputs for the short input should match.
										if scaling_type == "dynamic":
															self.assertTrue(torch.allclose(UpperCamelCase__							,					UpperCamelCase__							,					atol=1e-5       )       )
										else:
															self.assertFalse(torch.allclose(UpperCamelCase__							,					UpperCamelCase__							,					atol=1e-5       )       )

										# The output should be different for long inputs
										self.assertFalse(torch.allclose(UpperCamelCase__							,					UpperCamelCase__							,					atol=1e-5       )       )


