
"""simple docstring"""







import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    GPTNeoXForCausalLM,
		    GPTNeoXForQuestionAnswering,
		    GPTNeoXForSequenceClassification,
		    GPTNeoXForTokenClassification,
		    GPTNeoXModel,
		)







class 							__magic_name__     :





					def __init__(				self   :							str							,						snake_case_   :							Dict							,						snake_case_   :							Union[str, Any]=13							,						snake_case_   :							List[Any]=7							,						snake_case_   :							int=True							,						snake_case_   :							Union[str, Any]=True							,						snake_case_   :							Dict=True							,						snake_case_   :							Union[str, Any]=True							,						snake_case_   :							List[Any]=99							,						snake_case_   :							Dict=64							,						snake_case_   :							Tuple=5							,						snake_case_   :							Optional[int]=4							,						snake_case_   :							List[Any]=37							,						snake_case_   :							List[Any]="gelu"							,						snake_case_   :							Dict=0.1							,						snake_case_   :							Dict=0.1							,						snake_case_   :							Optional[int]=512							,						snake_case_   :							str=16							,						snake_case_   :							Dict=2							,						snake_case_   :							Tuple=0.02							,						snake_case_   :							Optional[Any]=3							,						snake_case_   :							List[str]=4							,						snake_case_   :							List[Any]=None							,						):
												__snake_case     			=       parent
												__snake_case     			=       batch_size
												__snake_case     			=       seq_length
												__snake_case     			=       is_training
												__snake_case     			=       use_input_mask
												__snake_case     			=       use_token_type_ids
												__snake_case     			=       use_labels
												__snake_case     			=       vocab_size
												__snake_case     			=       hidden_size
												__snake_case     			=       num_hidden_layers
												__snake_case     			=       num_attention_heads
												__snake_case     			=       intermediate_size
												__snake_case     			=       hidden_act
												__snake_case     			=       hidden_dropout_prob
												__snake_case     			=       attention_probs_dropout_prob
												__snake_case     			=       max_position_embeddings
												__snake_case     			=       type_vocab_size
												__snake_case     			=       type_sequence_label_size
												__snake_case     			=       initializer_range
												__snake_case     			=       num_labels
												__snake_case     			=       num_choices
												__snake_case     			=       scope
												__snake_case     			=       vocab_size - 1





					def 							lowerCAmelCase      (				self   :							Optional[int]    ):
												__snake_case     			=       ids_tensor([self.batch_size, self.seq_length]							,						self.vocab_size    )

												__snake_case     			=       None
												if self.use_input_mask:
																			__snake_case     			=       random_attention_mask([self.batch_size, self.seq_length]    )

												__snake_case     			=       None
												if self.use_labels:
																			__snake_case     			=       ids_tensor([self.batch_size, self.seq_length]							,						self.num_labels    )

												__snake_case     			=       self.get_config()

												return config, input_ids, input_mask, token_labels





					def 							lowerCAmelCase      (				self   :							Tuple    ):
												return GPTNeoXConfig(
												    vocab_size=self.vocab_size							,						hidden_size=self.hidden_size							,						num_hidden_layers=self.num_hidden_layers							,						num_attention_heads=self.num_attention_heads							,						intermediate_size=self.intermediate_size							,						hidden_act=self.hidden_act							,						hidden_dropout_prob=self.hidden_dropout_prob							,						attention_probs_dropout_prob=self.attention_probs_dropout_prob							,						max_position_embeddings=self.max_position_embeddings							,						type_vocab_size=self.type_vocab_size							,						is_decoder=snake_case_							,						initializer_range=self.initializer_range							,						pad_token_id=self.pad_token_id							,						)





					def 							lowerCAmelCase      (				self   :							List[str]    ):
												__snake_case			,   __snake_case			,   __snake_case			,   __snake_case     			=       self.prepare_config_and_inputs()

												__snake_case     			=       True

												return config, input_ids, input_mask, token_labels





					def 							lowerCAmelCase      (				self   :							Tuple							,						snake_case_   :							List[Any]							,						snake_case_   :							str							,						snake_case_   :							int    ):
												__snake_case     			=       GPTNeoXModel(config=snake_case_    )
												model.to(snake_case_    )
												model.eval()
												__snake_case     			=       model(snake_case_							,						attention_mask=snake_case_    )
												__snake_case     			=       model(snake_case_    )
												self.parent.assertEqual(result.last_hidden_state.shape							,						(self.batch_size, self.seq_length, self.hidden_size)    )





					def 							lowerCAmelCase      (				self   :							List[str]							,						snake_case_   :							str							,						snake_case_   :							Any							,						snake_case_   :							Optional[int]    ):
												__snake_case     			=       True
												__snake_case     			=       GPTNeoXModel(snake_case_    )
												model.to(snake_case_    )
												model.eval()
												__snake_case     			=       model(snake_case_							,						attention_mask=snake_case_    )
												self.parent.assertEqual(result.last_hidden_state.shape							,						(self.batch_size, self.seq_length, self.hidden_size)    )





					def 							lowerCAmelCase      (				self   :							List[Any]							,						snake_case_   :							Tuple							,						snake_case_   :							List[Any]							,						snake_case_   :							int							,						snake_case_   :							Optional[Any]    ):
												__snake_case     			=       GPTNeoXForCausalLM(config=snake_case_    )
												model.to(snake_case_    )
												model.eval()
												__snake_case     			=       model(snake_case_							,						attention_mask=snake_case_							,						labels=snake_case_    )
												self.parent.assertEqual(result.logits.shape							,						(self.batch_size, self.seq_length, self.vocab_size)    )





					def 							lowerCAmelCase      (				self   :							List[Any]							,						snake_case_   :							Optional[int]							,						snake_case_   :							Optional[Any]							,						snake_case_   :							str							,						snake_case_   :							int    ):
												__snake_case     			=       self.num_labels
												__snake_case     			=       GPTNeoXForQuestionAnswering(snake_case_    )
												model.to(snake_case_    )
												model.eval()
												__snake_case     			=       model(snake_case_							,						attention_mask=snake_case_    )
												self.parent.assertEqual(result.start_logits.shape							,						(self.batch_size, self.seq_length)    )
												self.parent.assertEqual(result.end_logits.shape							,						(self.batch_size, self.seq_length)    )





					def 							lowerCAmelCase      (				self   :							Tuple							,						snake_case_   :							int							,						snake_case_   :							Tuple							,						snake_case_   :							Any							,						snake_case_   :							Optional[Any]    ):
												__snake_case     			=       self.num_labels
												__snake_case     			=       GPTNeoXForSequenceClassification(snake_case_    )
												model.to(snake_case_    )
												model.eval()
												__snake_case     			=       ids_tensor([self.batch_size]							,						self.type_sequence_label_size    )
												__snake_case     			=       model(snake_case_							,						attention_mask=snake_case_							,						labels=snake_case_    )
												self.parent.assertEqual(result.logits.shape							,						(self.batch_size, self.num_labels)    )





					def 							lowerCAmelCase      (				self   :							Union[str, Any]							,						snake_case_   :							Tuple							,						snake_case_   :							Tuple							,						snake_case_   :							Optional[Any]							,						snake_case_   :							Dict    ):
												__snake_case     			=       self.num_labels
												__snake_case     			=       GPTNeoXForTokenClassification(snake_case_    )
												model.to(snake_case_    )
												model.eval()
												__snake_case     			=       model(snake_case_							,						attention_mask=snake_case_							,						labels=snake_case_    )
												self.parent.assertEqual(result.logits.shape							,						(self.batch_size, self.seq_length, self.num_labels)    )





					def 							lowerCAmelCase      (				self   :							List[Any]							,						snake_case_   :							Optional[int]							,						snake_case_   :							Union[str, Any]							,						snake_case_   :							Tuple    ):
												__snake_case     			=       True
												__snake_case     			=       GPTNeoXForCausalLM(config=snake_case_    )
												model.to(snake_case_    )
												model.eval()

												# first forward pass
												__snake_case     			=       model(snake_case_							,						attention_mask=snake_case_							,						use_cache=snake_case_    )
												__snake_case     			=       outputs.past_key_values

												# create hypothetical multiple next token and extent to next_input_ids
												__snake_case     			=       ids_tensor((self.batch_size, 3)							,						config.vocab_size    )
												__snake_case     			=       ids_tensor((self.batch_size, 3)							,						vocab_size=2    )

												# append to next input_ids and
												__snake_case     			=       torch.cat([input_ids, next_tokens]							,						dim=-1    )
												__snake_case     			=       torch.cat([input_mask, next_mask]							,						dim=-1    )

												__snake_case     			=       model(snake_case_							,						attention_mask=snake_case_							,						output_hidden_states=snake_case_    )
												__snake_case     			=       output_from_no_past["hidden_states"][0]
												__snake_case     			=       model(
												    snake_case_							,						attention_mask=snake_case_							,						past_key_values=snake_case_							,						output_hidden_states=snake_case_							,						)["hidden_states"][0]

												# select random slice
												__snake_case     			=       ids_tensor((1,)							,						output_from_past.shape[-1]    ).item()
												__snake_case     			=       output_from_no_past[:, -3:, random_slice_idx].detach()
												__snake_case     			=       output_from_past[:, :, random_slice_idx].detach()

												self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]    )

												# test that outputs are equal for slice
												self.parent.assertTrue(torch.allclose(snake_case_							,						snake_case_							,						atol=1e-3    )    )





					def 							lowerCAmelCase      (				self   :							int    ):
												__snake_case     			=       self.prepare_config_and_inputs()
												__snake_case			,   __snake_case			,   __snake_case			,   __snake_case     			=       config_and_inputs
												__snake_case     			=       {"input_ids": input_ids, "attention_mask": input_mask}
												return config, inputs_dict







@require_torch
class 							__magic_name__     (		lowercase__      ,		lowercase__      ,		lowercase__      ,		unittest.TestCase   ):
					_SCREAMING_SNAKE_CASE       :					Optional[int]     							=   (
					    (
					        GPTNeoXModel,
					        GPTNeoXForCausalLM,
					        GPTNeoXForQuestionAnswering,
					        GPTNeoXForSequenceClassification,
					        GPTNeoXForTokenClassification,
					    )
					    if is_torch_available()
					    else ()
					)
					_SCREAMING_SNAKE_CASE       :					Any     							=   (GPTNeoXForCausalLM,) if is_torch_available() else ()
					_SCREAMING_SNAKE_CASE       :					Optional[int]     							=   (
					    {
					        'feature-extraction': GPTNeoXModel,
					        'question-answering': GPTNeoXForQuestionAnswering,
					        'text-classification': GPTNeoXForSequenceClassification,
					        'text-generation': GPTNeoXForCausalLM,
					        'token-classification': GPTNeoXForTokenClassification,
					        'zero-shot': GPTNeoXForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					_SCREAMING_SNAKE_CASE       :					Optional[Any]     							=   False
					_SCREAMING_SNAKE_CASE       :					int     							=   False
					_SCREAMING_SNAKE_CASE       :					str     							=   False
					_SCREAMING_SNAKE_CASE       :					Optional[int]     							=   False





					def 							lowerCAmelCase      (				self   :							Tuple    ):
												__snake_case     			=       GPTNeoXModelTester(self    )
												__snake_case     			=       ConfigTester(self							,						config_class=snake_case_							,						hidden_size=64							,						num_attention_heads=8    )





					def 							lowerCAmelCase      (				self   :							Union[str, Any]    ):
												self.config_tester.run_common_tests()





					def 							lowerCAmelCase      (				self   :							int    ):
												__snake_case			,   __snake_case			,   __snake_case			,   __snake_case     			=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(snake_case_							,						snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							Dict    ):
												__snake_case			,   __snake_case			,   __snake_case			,   __snake_case     			=       self.model_tester.prepare_config_and_inputs_for_decoder()
												self.model_tester.create_and_check_model_as_decoder(snake_case_							,						snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							List[Any]    ):
												# This regression test was failing with PyTorch < 1.3
												__snake_case			,   __snake_case			,   __snake_case			,   __snake_case     			=       self.model_tester.prepare_config_and_inputs_for_decoder()

												__snake_case     			=       None

												self.model_tester.create_and_check_model_as_decoder(snake_case_							,						snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							Any    ):
												__snake_case			,   __snake_case			,   __snake_case			,   __snake_case     			=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_decoder_model_past_large_inputs(snake_case_							,						snake_case_							,						snake_case_    )





					def 							lowerCAmelCase      (				self   :							int    ):
												__snake_case     			=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_causal_lm(*snake_case_    )





					def 							lowerCAmelCase      (				self   :							Union[str, Any]    ):
												__snake_case     			=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*snake_case_    )





					def 							lowerCAmelCase      (				self   :							str    ):
												__snake_case     			=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*snake_case_    )





					def 							lowerCAmelCase      (				self   :							Optional[int]    ):
												__snake_case     			=       self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*snake_case_    )





					@unittest.skip(reason="Feed forward chunking is not implemented"    )
					def 							lowerCAmelCase      (				self   :							Any    ):
												pass





					@parameterized.expand([("linear",), ("dynamic",)]    )
					def 							lowerCAmelCase      (				self   :							Optional[int]							,						snake_case_   :							List[str]    ):
												__snake_case			,   __snake_case     			=       self.model_tester.prepare_config_and_inputs_for_common()
												__snake_case     			=       ids_tensor([1, 10]							,						config.vocab_size    )
												__snake_case     			=       ids_tensor([1, int(config.max_position_embeddings * 1.5    )]							,						config.vocab_size    )

												set_seed(42    )  # Fixed seed at init time so the two models get the same random weights
												__snake_case     			=       GPTNeoXModel(snake_case_    )
												original_model.to(snake_case_    )
												original_model.eval()
												__snake_case     			=       original_model(snake_case_    ).last_hidden_state
												__snake_case     			=       original_model(snake_case_    ).last_hidden_state

												set_seed(42    )  # Fixed seed at init time so the two models get the same random weights
												__snake_case     			=       {"type": scaling_type, "factor": 10.0}
												__snake_case     			=       GPTNeoXModel(snake_case_    )
												scaled_model.to(snake_case_    )
												scaled_model.eval()
												__snake_case     			=       scaled_model(snake_case_    ).last_hidden_state
												__snake_case     			=       scaled_model(snake_case_    ).last_hidden_state

												# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
												# maximum sequence length, so the outputs for the short input should match.
												if scaling_type == "dynamic":
																			self.assertTrue(torch.allclose(snake_case_							,						snake_case_							,						atol=1e-5    )    )
												else:
																			self.assertFalse(torch.allclose(snake_case_							,						snake_case_							,						atol=1e-5    )    )

												# The output should be different for long inputs
												self.assertFalse(torch.allclose(snake_case_							,						snake_case_							,						atol=1e-5    )    )







@require_torch
class 							__magic_name__     (		unittest.TestCase   ):





					@slow
					def 							lowerCAmelCase      (				self   :							Optional[Any]    ):
												__snake_case     			=       AutoTokenizer.from_pretrained("EleutherAI/pythia-410m-deduped"    )
												for checkpointing in [True, False]:
																			__snake_case     			=       GPTNeoXForCausalLM.from_pretrained("EleutherAI/pythia-410m-deduped"    )

																			if checkpointing:
																										model.gradient_checkpointing_enable()
																			else:
																										model.gradient_checkpointing_disable()
																			model.to(snake_case_    )

																			__snake_case     			=       tokenizer("My favorite food is"							,						return_tensors="pt"    ).to(snake_case_    )
																			# The hub repo. is updated on 2023-04-04, resulting in poor outputs.
																			# See: https://github.com/huggingface/transformers/pull/24193
																			__snake_case     			=       "My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI'm not sure"

																			__snake_case     			=       model.generate(**snake_case_							,						do_sample=snake_case_							,						max_new_tokens=20    )
																			__snake_case     			=       tokenizer.batch_decode(snake_case_    )[0]

																			self.assertEqual(snake_case_							,						snake_case_    )


"""simple docstring"""







from __future__ import annotations

from collections.abc import Callable
def   __UpperCamelCase   (					SCREAMING_SNAKE_CASE							,    SCREAMING_SNAKE_CASE							,    SCREAMING_SNAKE_CASE							,    SCREAMING_SNAKE_CASE = 1_00							,    )  ->     float:







							"""simple docstring"""


							__snake_case     			=       x_start
							__snake_case     			=       fnc(SCREAMING_SNAKE_CASE )
							__snake_case     			=       0.0
							for _ in range(SCREAMING_SNAKE_CASE ):
														# Approximates small segments of curve as linear and solve
														# for trapezoidal area
														__snake_case     			=       (x_end - x_start) / steps + xa
														__snake_case     			=       fnc(SCREAMING_SNAKE_CASE )
														area += abs(fxa + fxa ) * (xa - xa) / 2
														# Increment step
														__snake_case     			=       xa
														__snake_case     			=       fxa
							return area


if __name__ == "__main__":
		def   __UpperCamelCase   (					SCREAMING_SNAKE_CASE )  ->     List[Any]:







									"""simple docstring"""


									return x**3 + x**2

		print("""f(x) = x^3 + x^2""")
		print("""The area between the curve, x = -5, x = 5 and the x axis is:""")
		_SCREAMING_SNAKE_CASE	       =      10
		while i <= 100_000:
				print(F"""with {i} steps: {trapezoidal_area(f, -5, 5, i)}""")
				i *= 10
