



import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    FalconForCausalLM,
			    FalconForQuestionAnswering,
			    FalconForSequenceClassification,
			    FalconForTokenClassification,
			    FalconModel,
			)



class 						UpperCAmelCase_					:







							def __init__(     self,    __a,    __a=3,    __a=7,    __a=True,    __a=True,    __a=False,    __a=True,    __a=99,    __a=32,    __a=5,    __a=4,    __a=37,    __a="gelu",    __a=0.1,    __a=0.1,    __a=512,    __a=16,    __a=2,    __a=0.02,    __a=3,    __a=4,    __a=None,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							List[str]				     =					parent
														_lowerCAmelCase  :							Any				     =					batch_size
														_lowerCAmelCase  :							Union[str, Any]				     =					seq_length
														_lowerCAmelCase  :							Dict				     =					is_training
														_lowerCAmelCase  :							List[Any]				     =					use_input_mask
														_lowerCAmelCase  :							Tuple				     =					use_token_type_ids
														_lowerCAmelCase  :							Optional[Any]				     =					use_labels
														_lowerCAmelCase  :							Union[str, Any]				     =					vocab_size
														_lowerCAmelCase  :							Dict				     =					hidden_size
														_lowerCAmelCase  :							int				     =					num_hidden_layers
														_lowerCAmelCase  :							List[Any]				     =					num_attention_heads
														_lowerCAmelCase  :							int				     =					intermediate_size
														_lowerCAmelCase  :							Tuple				     =					hidden_act
														_lowerCAmelCase  :							Tuple				     =					hidden_dropout_prob
														_lowerCAmelCase  :							Optional[int]				     =					attention_probs_dropout_prob
														_lowerCAmelCase  :							Tuple				     =					max_position_embeddings
														_lowerCAmelCase  :							str				     =					type_vocab_size
														_lowerCAmelCase  :							Optional[int]				     =					type_sequence_label_size
														_lowerCAmelCase  :							str				     =					initializer_range
														_lowerCAmelCase  :							Tuple				     =					num_labels
														_lowerCAmelCase  :							Union[str, Any]				     =					num_choices
														_lowerCAmelCase  :							str				     =					scope







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							List[str]				     =					ids_tensor([self.batch_size, self.seq_length],    self.vocab_size)

														_lowerCAmelCase  :							Optional[Any]				     =					None
														if self.use_input_mask:
																					_lowerCAmelCase  :							Dict				     =					random_attention_mask([self.batch_size, self.seq_length])

														_lowerCAmelCase  :							Union[str, Any]				     =					None

														_lowerCAmelCase  :							Optional[Any]				     =					None
														_lowerCAmelCase  :							Tuple				     =					None
														_lowerCAmelCase  :							Tuple				     =					None
														if self.use_labels:
																					_lowerCAmelCase  :							Optional[Any]				     =					ids_tensor([self.batch_size],    self.type_sequence_label_size)
																					_lowerCAmelCase  :							Any				     =					ids_tensor([self.batch_size, self.seq_length],    self.num_labels)
																					_lowerCAmelCase  :							Optional[Any]				     =					ids_tensor([self.batch_size],    self.num_choices)

														_lowerCAmelCase  :							int				     =					self.get_config()

														return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







							def      snake_case__   (     self):

														'''simple docstring'''

														return FalconConfig(
														    vocab_size=self.vocab_size,    hidden_size=self.hidden_size,    num_hidden_layers=self.num_hidden_layers,    num_attention_heads=self.num_attention_heads,    intermediate_size=self.intermediate_size,    hidden_act=self.hidden_act,    hidden_dropout_prob=self.hidden_dropout_prob,    attention_probs_dropout_prob=self.attention_probs_dropout_prob,    max_position_embeddings=self.max_position_embeddings,    type_vocab_size=self.type_vocab_size,    is_decoder=_lowerCamelCase,    initializer_range=self.initializer_range,    pad_token_id=1,    new_decoder_architecture=_lowerCamelCase,    )







							def      snake_case__   (     self,    __a,    __a,    __a,    __a,    __a,    __a,    __a):

														'''simple docstring'''

														_lowerCAmelCase  :							Union[str, Any]				     =					FalconModel(config=_lowerCamelCase)
														model.to(_lowerCamelCase)
														model.eval()
														_lowerCAmelCase  :							Any				     =					model(_lowerCamelCase,    attention_mask=_lowerCamelCase)
														_lowerCAmelCase  :							Optional[Any]				     =					model(_lowerCamelCase)
														self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.hidden_size))







							def      snake_case__   (     self,    __a,    __a,    __a,    __a,    __a,    __a,    __a,    __a,    __a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							List[Any]				     =					True
														_lowerCAmelCase  :							Tuple				     =					FalconModel(_lowerCamelCase)
														model.to(_lowerCamelCase)
														model.eval()
														_lowerCAmelCase  :							Dict				     =					model(
														    _lowerCamelCase,    attention_mask=_lowerCamelCase,    encoder_hidden_states=_lowerCamelCase,    encoder_attention_mask=_lowerCamelCase,    )
														_lowerCAmelCase  :							List[str]				     =					model(
														    _lowerCamelCase,    attention_mask=_lowerCamelCase,    encoder_hidden_states=_lowerCamelCase,    )
														_lowerCAmelCase  :							Optional[int]				     =					model(_lowerCamelCase,    attention_mask=_lowerCamelCase)
														self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.hidden_size))







							def      snake_case__   (     self,    __a,    __a,    __a,    __a,    __a,    __a,    __a,    __a,    __a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							Optional[int]				     =					FalconForCausalLM(config=_lowerCamelCase)
														model.to(_lowerCamelCase)
														model.eval()
														_lowerCAmelCase  :							List[str]				     =					model(_lowerCamelCase,    attention_mask=_lowerCamelCase,    labels=_lowerCamelCase)
														self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.seq_length, self.vocab_size))







							def      snake_case__   (     self,    __a,    __a,    __a,    __a,    __a,    __a,    __a,    __a,    __a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							Dict				     =					True
														_lowerCAmelCase  :							int				     =					True
														_lowerCAmelCase  :							Any				     =					FalconForCausalLM(config=_lowerCamelCase)
														model.to(_lowerCamelCase)
														model.eval()

														# first forward pass
														_lowerCAmelCase  :							Any				     =					model(
														    _lowerCamelCase,    attention_mask=_lowerCamelCase,    encoder_hidden_states=_lowerCamelCase,    encoder_attention_mask=_lowerCamelCase,    use_cache=_lowerCamelCase,    )
														_lowerCAmelCase  :							Union[str, Any]				     =					outputs.past_key_values

														# create hypothetical multiple next token and extent to next_input_ids
														_lowerCAmelCase  :							Dict				     =					ids_tensor((self.batch_size, 3),    config.vocab_size)
														_lowerCAmelCase  :							List[str]				     =					ids_tensor((self.batch_size, 3),    vocab_size=2)

														# append to next input_ids and
														_lowerCAmelCase  :							str				     =					torch.cat([input_ids, next_tokens],    dim=-1)
														_lowerCAmelCase  :							Tuple				     =					torch.cat([input_mask, next_mask],    dim=-1)

														_lowerCAmelCase  :							Union[str, Any]				     =					model(
														    _lowerCamelCase,    attention_mask=_lowerCamelCase,    encoder_hidden_states=_lowerCamelCase,    encoder_attention_mask=_lowerCamelCase,    output_hidden_states=_lowerCamelCase,    )["hidden_states"][0]
														_lowerCAmelCase  :							Any				     =					model(
														    _lowerCamelCase,    attention_mask=_lowerCamelCase,    encoder_hidden_states=_lowerCamelCase,    encoder_attention_mask=_lowerCamelCase,    past_key_values=_lowerCamelCase,    output_hidden_states=_lowerCamelCase,    )["hidden_states"][0]

														# select random slice
														_lowerCAmelCase  :							Optional[int]				     =					ids_tensor((1,),    output_from_past.shape[-1]).item()
														_lowerCAmelCase  :							str				     =					output_from_no_past[:, -3:, random_slice_idx].detach()
														_lowerCAmelCase  :							Tuple				     =					output_from_past[:, :, random_slice_idx].detach()

														self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

														# test that outputs are equal for slice
														self.parent.assertTrue(torch.allclose(_lowerCamelCase,    _lowerCamelCase,    atol=1E-3))







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							Optional[Any]				     =					self.prepare_config_and_inputs()
														(
														    (
														    _lowerCAmelCase	
														)     ,					(
														    _lowerCAmelCase	
														)     ,					(
														    _lowerCAmelCase	
														)     ,					(
														    _lowerCAmelCase	
														)     ,					(
														    _lowerCAmelCase	
														)     ,					(
														    _lowerCAmelCase	
														)     ,					(
														    _lowerCAmelCase	
														)     ,						
														)  :							Tuple				     =					config_and_inputs
														_lowerCAmelCase  :							Optional[int]				     =					{"input_ids": input_ids, "attention_mask": input_mask}
														return config, inputs_dict



@require_torch
class 						UpperCAmelCase_					( __UpperCAmelCase     ,      __UpperCAmelCase     ,      __UpperCAmelCase     ,      unittest.TestCase):
							lowerCamelCase__         =	(
							    (
							        FalconModel,
							        FalconForCausalLM,
							        FalconForSequenceClassification,
							        FalconForTokenClassification,
							        FalconForQuestionAnswering,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCamelCase__         =	(FalconForCausalLM,) if is_torch_available() else ()
							lowerCamelCase__         =	(
							    {
							        "feature-extraction": FalconModel,
							        "text-classification": FalconForSequenceClassification,
							        "text-generation": FalconForCausalLM,
							        "question-answering": FalconForQuestionAnswering,
							        "token-classification": FalconForTokenClassification,
							        "zero-shot": FalconForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							lowerCamelCase__         =	False
							lowerCamelCase__         =	False







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							List[Any]				     =					FalconModelTester(self)
														_lowerCAmelCase  :							Any				     =					ConfigTester(self,    config_class=_lowerCamelCase,    hidden_size=37)







							def      snake_case__   (     self):

														'''simple docstring'''

														self.config_tester.run_common_tests()







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							Union[str, Any]				     =					self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*_lowerCamelCase)







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase     ,					*_lowerCAmelCase  :							Dict				     =					self.model_tester.prepare_config_and_inputs()
														for alibi in [True, False]:
																					_lowerCAmelCase  :							List[str]				     =					alibi
																					self.model_tester.create_and_check_model(_lowerCamelCase,    *_lowerCamelCase)







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase     ,					_lowerCAmelCase  :							Dict				     =					self.model_tester.prepare_config_and_inputs_for_common()
														_lowerCAmelCase  :							Optional[int]				     =					3
														_lowerCAmelCase  :							Optional[int]				     =					input_dict["input_ids"]
														_lowerCAmelCase  :							Dict				     =					input_ids.ne(1).to(_lowerCamelCase)
														_lowerCAmelCase  :							Union[str, Any]				     =					ids_tensor([self.model_tester.batch_size],    self.model_tester.type_sequence_label_size)
														_lowerCAmelCase  :							List[str]				     =					FalconForSequenceClassification(_lowerCamelCase)
														model.to(_lowerCamelCase)
														model.eval()
														_lowerCAmelCase  :							Optional[Any]				     =					model(_lowerCamelCase,    attention_mask=_lowerCamelCase,    labels=_lowerCamelCase)
														self.assertEqual(result.logits.shape,    (self.model_tester.batch_size, self.model_tester.num_labels))







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase     ,					_lowerCAmelCase  :							int				     =					self.model_tester.prepare_config_and_inputs_for_common()
														_lowerCAmelCase  :							Optional[Any]				     =					3
														_lowerCAmelCase  :							str				     =					"single_label_classification"
														_lowerCAmelCase  :							List[str]				     =					input_dict["input_ids"]
														_lowerCAmelCase  :							Optional[Any]				     =					input_ids.ne(1).to(_lowerCamelCase)
														_lowerCAmelCase  :							Optional[int]				     =					ids_tensor([self.model_tester.batch_size],    self.model_tester.type_sequence_label_size)
														_lowerCAmelCase  :							Tuple				     =					FalconForSequenceClassification(_lowerCamelCase)
														model.to(_lowerCamelCase)
														model.eval()
														_lowerCAmelCase  :							Union[str, Any]				     =					model(_lowerCamelCase,    attention_mask=_lowerCamelCase,    labels=_lowerCamelCase)
														self.assertEqual(result.logits.shape,    (self.model_tester.batch_size, self.model_tester.num_labels))







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase     ,					_lowerCAmelCase  :							int				     =					self.model_tester.prepare_config_and_inputs_for_common()
														_lowerCAmelCase  :							str				     =					input_dict["input_ids"]
														_lowerCAmelCase  :							str				     =					FalconForCausalLM(_lowerCamelCase)
														model.to(_lowerCamelCase)
														model.eval()
														_lowerCAmelCase  :							Tuple				     =					model(_lowerCamelCase,    use_cache=_lowerCamelCase)
														_lowerCAmelCase  :							Optional[int]				     =					input_ids.shape[0]
														_lowerCAmelCase  :							Dict				     =					model._convert_to_rw_cache(result.past_key_values)
														_lowerCAmelCase  :							List[Any]				     =					model._convert_cache_to_standard_format(_lowerCamelCase,    _lowerCamelCase)
														for layer in range(len(_lowerCamelCase)):
																					for tensor_idx in range(2):
																												self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3)
																												self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4)
																												self.assertTrue(
																												    torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]))







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase     ,					_lowerCAmelCase  :							List[Any]				     =					self.model_tester.prepare_config_and_inputs_for_common()
														_lowerCAmelCase  :							Any				     =					3
														_lowerCAmelCase  :							str				     =					"multi_label_classification"
														_lowerCAmelCase  :							Tuple				     =					input_dict["input_ids"]
														_lowerCAmelCase  :							int				     =					input_ids.ne(1).to(_lowerCamelCase)
														_lowerCAmelCase  :							Union[str, Any]				     =					ids_tensor(
														    [self.model_tester.batch_size, config.num_labels],    self.model_tester.type_sequence_label_size).to(torch.float)
														_lowerCAmelCase  :							int				     =					FalconForSequenceClassification(_lowerCamelCase)
														model.to(_lowerCamelCase)
														model.eval()
														_lowerCAmelCase  :							List[str]				     =					model(_lowerCamelCase,    attention_mask=_lowerCamelCase,    labels=_lowerCamelCase)
														self.assertEqual(result.logits.shape,    (self.model_tester.batch_size, self.model_tester.num_labels))







							def      snake_case__   (     self):

														'''simple docstring'''

														for model_class in self.all_generative_model_classes:
																					_lowerCAmelCase     ,					_lowerCAmelCase  :							List[Any]				     =					self.model_tester.prepare_config_and_inputs_for_common()

																					# If it doesn't support cache, pass the test
																					if not hasattr(_lowerCamelCase,    "use_cache"):
																												return

																					_lowerCAmelCase  :							int				     =					model_class(_lowerCamelCase).to(_lowerCamelCase)
																					if "use_cache" not in inputs:
																												_lowerCAmelCase  :							Dict				     =					True
																					_lowerCAmelCase  :							Optional[int]				     =					model(**_lowerCamelCase)

																					# If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
																					if "past_key_values" not in outputs:
																												return

																					_lowerCAmelCase  :							List[str]				     =					(
																					    getattr(_lowerCamelCase,    "decoder_layers",    _lowerCamelCase)
																					    or getattr(_lowerCamelCase,    "num_decoder_layers",    _lowerCamelCase)
																					    or config.num_hidden_layers
																					)
																					_lowerCAmelCase  :							List[Any]				     =					getattr(_lowerCamelCase,    "num_kv_heads",    config.num_attention_heads)
																					_lowerCAmelCase  :							Optional[int]				     =					getattr(_lowerCamelCase,    "d_model",    config.hidden_size)
																					_lowerCAmelCase  :							List[str]				     =					embed_dim // num_attention_heads

																					_lowerCAmelCase  :							str				     =					outputs["past_key_values"]
																					self.assertEqual(len(_lowerCamelCase),    _lowerCamelCase)

																					_lowerCAmelCase     ,					_lowerCAmelCase  :							int				     =					inputs["input_ids"].shape
																					for i in range(_lowerCamelCase):
																												if config.new_decoder_architecture:
																																			_lowerCAmelCase  :							Any				     =					config.num_attention_heads
																												elif config.multi_query:
																																			_lowerCAmelCase  :							Optional[Any]				     =					1
																												self.assertEqual(len(past_kv[0]),    2)  # K V for the decoder = 2
																												self.assertEqual(
																												    past_kv[i][0].shape,    (batch_size, num_attention_heads, seq_length, per_head_embed_dim))
																												self.assertEqual(
																												    past_kv[i][1].shape,    (batch_size, num_attention_heads, seq_length, per_head_embed_dim))



@require_torch
class 						UpperCAmelCase_					( unittest.TestCase):







							@slow
							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							Union[str, Any]				     =					AutoTokenizer.from_pretrained("Rocketknight1/falcon-rw-1b")
														_lowerCAmelCase  :							int				     =					FalconForCausalLM.from_pretrained("Rocketknight1/falcon-rw-1b")
														model.eval()
														model.to(_lowerCamelCase)
														_lowerCAmelCase  :							Tuple				     =					tokenizer("My favorite food is",    return_tensors="pt").to(_lowerCamelCase)

														_lowerCAmelCase  :							int				     =					(
														    "My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday."
														)

														_lowerCAmelCase  :							Optional[Any]				     =					model.generate(**_lowerCamelCase,    do_sample=_lowerCamelCase,    max_new_tokens=19)
														_lowerCAmelCase  :							Tuple				     =					tokenizer.batch_decode(_lowerCamelCase)[0]

														self.assertEqual(_lowerCamelCase,    _lowerCamelCase)







							@slow
							def      snake_case__   (     self):

														'''simple docstring'''

														for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
																					_lowerCAmelCase  :							str				     =					AutoTokenizer.from_pretrained(_lowerCamelCase)
																					_lowerCAmelCase  :							List[str]				     =					FalconForCausalLM.from_pretrained(_lowerCamelCase)
																					model.eval()
																					model.to(_lowerCamelCase)
																					_lowerCAmelCase  :							Union[str, Any]				     =					tokenizer("My favorite food is",    return_tensors="pt").to(_lowerCamelCase)

																					# We just test that these run without errors - the models are randomly initialized
																					# and so the actual text outputs will be garbage
																					model.generate(**_lowerCamelCase,    do_sample=_lowerCamelCase,    max_new_tokens=4)
																					model.generate(**_lowerCamelCase,    do_sample=_lowerCamelCase,    max_new_tokens=4)
																					model.generate(**_lowerCamelCase,    num_beams=2,    max_new_tokens=4)







							@slow
							def      snake_case__   (     self):

														'''simple docstring'''

														with torch.no_grad():
																					for repo in [
																					    "Rocketknight1/falcon-rw-1b",
																					    "Rocketknight1/tiny-random-falcon-7b",
																					    "Rocketknight1/tiny-random-falcon-40b",
																					]:
																												_lowerCAmelCase  :							int				     =					AutoTokenizer.from_pretrained(_lowerCamelCase)
																												_lowerCAmelCase  :							Optional[int]				     =					FalconForCausalLM.from_pretrained(_lowerCamelCase)
																												model.eval()
																												model.to(device=_lowerCamelCase)
																												_lowerCAmelCase  :							Tuple				     =					tokenizer("My favorite food is",    return_tensors="pt").to(_lowerCamelCase)

																												# Test results are the same with and without cache
																												_lowerCAmelCase  :							str				     =					model.generate(**_lowerCamelCase,    do_sample=_lowerCamelCase,    max_new_tokens=20,    use_cache=_lowerCamelCase)
																												_lowerCAmelCase  :							List[str]				     =					model.generate(**_lowerCamelCase,    do_sample=_lowerCamelCase,    max_new_tokens=20,    use_cache=_lowerCamelCase)
																												self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0)





from __future__ import annotations






def        A       (      _lowerCamelCase						):





							'''simple docstring'''




							if not nums:
														raise ValueError("List is empty"						)
							return sum(_lowerCamelCase						) / len(_lowerCamelCase						)


if __name__ == "__main__":
						import doctest

						doctest.testmod()
