



import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    FalconForCausalLM,
			    FalconForQuestionAnswering,
			    FalconForSequenceClassification,
			    FalconForTokenClassification,
			    FalconModel,
			)
class 			_lowerCAmelCase   :






			"""simple docstring"""





			def __init__(				self	:       Any			,     lowercase	:       Optional[Any]			,     lowercase	:       List[Any]=3			,     lowercase	:       Any=7			,     lowercase	:       Dict=True			,     lowercase	:       Optional[Any]=True			,     lowercase	:       List[Any]=False			,     lowercase	:       List[Any]=True			,     lowercase	:       List[Any]=99			,     lowercase	:       Dict=32			,     lowercase	:       Tuple=5			,     lowercase	:       str=4			,     lowercase	:       Optional[int]=37			,     lowercase	:       int="gelu"			,     lowercase	:       Tuple=0.1			,     lowercase	:       Union[str, Any]=0.1			,     lowercase	:       Union[str, Any]=512			,     lowercase	:       Tuple=16			,     lowercase	:       str=2			,     lowercase	:       Optional[int]=0.02			,     lowercase	:       str=3			,     lowercase	:       Any=4			,     lowercase	:       Optional[Any]=None			,     )						->					Dict:
									"""simple docstring"""







									__lowercase					=				parent
									__lowercase					=				batch_size
									__lowercase					=				seq_length
									__lowercase					=				is_training
									__lowercase					=				use_input_mask
									__lowercase					=				use_token_type_ids
									__lowercase					=				use_labels
									__lowercase					=				vocab_size
									__lowercase					=				hidden_size
									__lowercase					=				num_hidden_layers
									__lowercase					=				num_attention_heads
									__lowercase					=				intermediate_size
									__lowercase					=				hidden_act
									__lowercase					=				hidden_dropout_prob
									__lowercase					=				attention_probs_dropout_prob
									__lowercase					=				max_position_embeddings
									__lowercase					=				type_vocab_size
									__lowercase					=				type_sequence_label_size
									__lowercase					=				initializer_range
									__lowercase					=				num_labels
									__lowercase					=				num_choices
									__lowercase					=				scope





			def 		snake_case__			(				self	:       int   )						->					int:
									"""simple docstring"""







									__lowercase					=				ids_tensor([self.batch_size, self.seq_length]			,     self.vocab_size   )

									__lowercase					=				None
									if self.use_input_mask:
															__lowercase					=				random_attention_mask([self.batch_size, self.seq_length]   )

									__lowercase					=				None

									__lowercase					=				None
									__lowercase					=				None
									__lowercase					=				None
									if self.use_labels:
															__lowercase					=				ids_tensor([self.batch_size]			,     self.type_sequence_label_size   )
															__lowercase					=				ids_tensor([self.batch_size, self.seq_length]			,     self.num_labels   )
															__lowercase					=				ids_tensor([self.batch_size]			,     self.num_choices   )

									__lowercase					=				self.get_config()

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





			def 		snake_case__			(				self	:       Tuple   )						->					List[str]:
									"""simple docstring"""







									return FalconConfig(
									    vocab_size=self.vocab_size			,     hidden_size=self.hidden_size			,     num_hidden_layers=self.num_hidden_layers			,     num_attention_heads=self.num_attention_heads			,     intermediate_size=self.intermediate_size			,     hidden_act=self.hidden_act			,     hidden_dropout_prob=self.hidden_dropout_prob			,     attention_probs_dropout_prob=self.attention_probs_dropout_prob			,     max_position_embeddings=self.max_position_embeddings			,     type_vocab_size=self.type_vocab_size			,     is_decoder=lowercase			,     initializer_range=self.initializer_range			,     pad_token_id=1			,     new_decoder_architecture=lowercase			,     )





			def 		snake_case__			(				self	:       Tuple			,     lowercase	:       str			,     lowercase	:       Optional[Any]			,     lowercase	:       int			,     lowercase	:       List[str]			,     lowercase	:       Union[str, Any]			,     lowercase	:       int			,     lowercase	:       Dict   )						->					List[Any]:
									"""simple docstring"""







									__lowercase					=				FalconModel(config=lowercase   )
									model.to(lowercase   )
									model.eval()
									__lowercase					=				model(lowercase			,     attention_mask=lowercase   )
									__lowercase					=				model(lowercase   )
									self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)   )





			def 		snake_case__			(				self	:       Optional[int]			,     lowercase	:       Union[str, Any]			,     lowercase	:       Tuple			,     lowercase	:       Optional[Any]			,     lowercase	:       Optional[int]			,     lowercase	:       Union[str, Any]			,     lowercase	:       Optional[Any]			,     lowercase	:       str			,     lowercase	:       Dict			,     lowercase	:       Any			,     )						->					str:
									"""simple docstring"""







									__lowercase					=				True
									__lowercase					=				FalconModel(lowercase   )
									model.to(lowercase   )
									model.eval()
									__lowercase					=				model(
									    lowercase			,     attention_mask=lowercase			,     encoder_hidden_states=lowercase			,     encoder_attention_mask=lowercase			,     )
									__lowercase					=				model(
									    lowercase			,     attention_mask=lowercase			,     encoder_hidden_states=lowercase			,     )
									__lowercase					=				model(lowercase			,     attention_mask=lowercase   )
									self.parent.assertEqual(result.last_hidden_state.shape			,     (self.batch_size, self.seq_length, self.hidden_size)   )





			def 		snake_case__			(				self	:       Optional[Any]			,     lowercase	:       str			,     lowercase	:       Tuple			,     lowercase	:       Any			,     lowercase	:       Optional[Any]			,     lowercase	:       Union[str, Any]			,     lowercase	:       int			,     lowercase	:       Tuple			,     lowercase	:       str			,     lowercase	:       Optional[Any]			,     )						->					List[Any]:
									"""simple docstring"""







									__lowercase					=				FalconForCausalLM(config=lowercase   )
									model.to(lowercase   )
									model.eval()
									__lowercase					=				model(lowercase			,     attention_mask=lowercase			,     labels=lowercase   )
									self.parent.assertEqual(result.logits.shape			,     (self.batch_size, self.seq_length, self.vocab_size)   )





			def 		snake_case__			(				self	:       Optional[int]			,     lowercase	:       Optional[int]			,     lowercase	:       Dict			,     lowercase	:       Any			,     lowercase	:       List[str]			,     lowercase	:       List[str]			,     lowercase	:       Tuple			,     lowercase	:       Union[str, Any]			,     lowercase	:       Optional[Any]			,     lowercase	:       List[str]			,     )						->					str:
									"""simple docstring"""







									__lowercase					=				True
									__lowercase					=				True
									__lowercase					=				FalconForCausalLM(config=lowercase   )
									model.to(lowercase   )
									model.eval()

									# first forward pass
									__lowercase					=				model(
									    lowercase			,     attention_mask=lowercase			,     encoder_hidden_states=lowercase			,     encoder_attention_mask=lowercase			,     use_cache=lowercase			,     )
									__lowercase					=				outputs.past_key_values

									# create hypothetical multiple next token and extent to next_input_ids
									__lowercase					=				ids_tensor((self.batch_size, 3)			,     config.vocab_size   )
									__lowercase					=				ids_tensor((self.batch_size, 3)			,     vocab_size=2   )

									# append to next input_ids and
									__lowercase					=				torch.cat([input_ids, next_tokens]			,     dim=-1   )
									__lowercase					=				torch.cat([input_mask, next_mask]			,     dim=-1   )

									__lowercase					=				model(
									    lowercase			,     attention_mask=lowercase			,     encoder_hidden_states=lowercase			,     encoder_attention_mask=lowercase			,     output_hidden_states=lowercase			,     )["""hidden_states"""][0]
									__lowercase					=				model(
									    lowercase			,     attention_mask=lowercase			,     encoder_hidden_states=lowercase			,     encoder_attention_mask=lowercase			,     past_key_values=lowercase			,     output_hidden_states=lowercase			,     )["""hidden_states"""][0]

									# select random slice
									__lowercase					=				ids_tensor((1,)			,     output_from_past.shape[-1]   ).item()
									__lowercase					=				output_from_no_past[:, -3:, random_slice_idx].detach()
									__lowercase					=				output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(lowercase			,     lowercase			,     atol=1E-3   )   )







			def 		snake_case__			(				self	:       Optional[int]   )						->					Union[str, Any]:
									"""simple docstring"""







									__lowercase					=				self.prepare_config_and_inputs()
									(
									    (
									    __lowercase
									)	,			(
									    __lowercase
									)	,			(
									    __lowercase
									)	,			(
									    __lowercase
									)	,			(
									    __lowercase
									)	,			(
									    __lowercase
									)	,			(
									    __lowercase
									)	,			
									)					=				config_and_inputs
									__lowercase					=				{"""input_ids""": input_ids, """attention_mask""": input_mask}
									return config, inputs_dict





@require_torch
class 			_lowerCAmelCase   ( _UpperCAmelCase    ,					_UpperCAmelCase    ,					_UpperCAmelCase    ,					unittest.TestCase			):






			"""simple docstring"""





			lowercase__      :  str      =			(
			    (
			        FalconModel,
			        FalconForCausalLM,
			        FalconForSequenceClassification,
			        FalconForTokenClassification,
			        FalconForQuestionAnswering,
			    )
			    if is_torch_available()
			    else ()
			)
			lowercase__      :  Any      =			(FalconForCausalLM,) if is_torch_available() else ()
			lowercase__      :  Union[str, Any]      =			(
			    {
			        """feature-extraction""": FalconModel,
			        """text-classification""": FalconForSequenceClassification,
			        """text-generation""": FalconForCausalLM,
			        """question-answering""": FalconForQuestionAnswering,
			        """token-classification""": FalconForTokenClassification,
			        """zero-shot""": FalconForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			lowercase__      :  Dict      =			False
			lowercase__      :  Tuple      =			False





			def 		snake_case__			(				self	:       Union[str, Any]   )						->					List[Any]:
									"""simple docstring"""







									__lowercase					=				FalconModelTester(self   )
									__lowercase					=				ConfigTester(self			,     config_class=lowercase			,     hidden_size=37   )





			def 		snake_case__			(				self	:       Dict   )						->					List[str]:
									"""simple docstring"""







									self.config_tester.run_common_tests()





			def 		snake_case__			(				self	:       Optional[int]   )						->					List[str]:
									"""simple docstring"""







									__lowercase					=				self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*lowercase   )





			def 		snake_case__			(				self	:       str   )						->					Optional[Any]:
									"""simple docstring"""







									__lowercase	,			*__lowercase					=				self.model_tester.prepare_config_and_inputs()
									for alibi in [True, False]:
															__lowercase					=				alibi
															self.model_tester.create_and_check_model(lowercase			,     *lowercase   )





			def 		snake_case__			(				self	:       Union[str, Any]   )						->					str:
									"""simple docstring"""







									__lowercase	,			__lowercase					=				self.model_tester.prepare_config_and_inputs_for_common()
									__lowercase					=				3
									__lowercase					=				input_dict["""input_ids"""]
									__lowercase					=				input_ids.ne(1   ).to(lowercase   )
									__lowercase					=				ids_tensor([self.model_tester.batch_size]			,     self.model_tester.type_sequence_label_size   )
									__lowercase					=				FalconForSequenceClassification(lowercase   )
									model.to(lowercase   )
									model.eval()
									__lowercase					=				model(lowercase			,     attention_mask=lowercase			,     labels=lowercase   )
									self.assertEqual(result.logits.shape			,     (self.model_tester.batch_size, self.model_tester.num_labels)   )





			def 		snake_case__			(				self	:       List[str]   )						->					Dict:
									"""simple docstring"""







									__lowercase	,			__lowercase					=				self.model_tester.prepare_config_and_inputs_for_common()
									__lowercase					=				3
									__lowercase					=				"""single_label_classification"""
									__lowercase					=				input_dict["""input_ids"""]
									__lowercase					=				input_ids.ne(1   ).to(lowercase   )
									__lowercase					=				ids_tensor([self.model_tester.batch_size]			,     self.model_tester.type_sequence_label_size   )
									__lowercase					=				FalconForSequenceClassification(lowercase   )
									model.to(lowercase   )
									model.eval()
									__lowercase					=				model(lowercase			,     attention_mask=lowercase			,     labels=lowercase   )
									self.assertEqual(result.logits.shape			,     (self.model_tester.batch_size, self.model_tester.num_labels)   )





			def 		snake_case__			(				self	:       Tuple   )						->					Optional[Any]:
									"""simple docstring"""







									__lowercase	,			__lowercase					=				self.model_tester.prepare_config_and_inputs_for_common()
									__lowercase					=				input_dict["""input_ids"""]
									__lowercase					=				FalconForCausalLM(lowercase   )
									model.to(lowercase   )
									model.eval()
									__lowercase					=				model(lowercase			,     use_cache=lowercase   )
									__lowercase					=				input_ids.shape[0]
									__lowercase					=				model._convert_to_rw_cache(result.past_key_values   )
									__lowercase					=				model._convert_cache_to_standard_format(lowercase			,     lowercase   )
									for layer in range(len(lowercase   )   ):
															for tensor_idx in range(2   ):
																					self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3   )
																					self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4   )
																					self.assertTrue(
																					    torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]   )   )





			def 		snake_case__			(				self	:       str   )						->					Union[str, Any]:
									"""simple docstring"""







									__lowercase	,			__lowercase					=				self.model_tester.prepare_config_and_inputs_for_common()
									__lowercase					=				3
									__lowercase					=				"""multi_label_classification"""
									__lowercase					=				input_dict["""input_ids"""]
									__lowercase					=				input_ids.ne(1   ).to(lowercase   )
									__lowercase					=				ids_tensor(
									    [self.model_tester.batch_size, config.num_labels]			,     self.model_tester.type_sequence_label_size   ).to(torch.float   )
									__lowercase					=				FalconForSequenceClassification(lowercase   )
									model.to(lowercase   )
									model.eval()
									__lowercase					=				model(lowercase			,     attention_mask=lowercase			,     labels=lowercase   )
									self.assertEqual(result.logits.shape			,     (self.model_tester.batch_size, self.model_tester.num_labels)   )







			def 		snake_case__			(				self	:       Union[str, Any]   )						->					Any:
									"""simple docstring"""







									for model_class in self.all_generative_model_classes:
															__lowercase	,			__lowercase					=				self.model_tester.prepare_config_and_inputs_for_common()

															# If it doesn't support cache, pass the test
															if not hasattr(lowercase			,     """use_cache"""   ):
																					return

															__lowercase					=				model_class(lowercase   ).to(lowercase   )
															if "use_cache" not in inputs:
																					__lowercase					=				True
															__lowercase					=				model(**lowercase   )

															# If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
															if "past_key_values" not in outputs:
																					return

															__lowercase					=				(
															    getattr(lowercase			,     """decoder_layers"""			,     lowercase   )
															    or getattr(lowercase			,     """num_decoder_layers"""			,     lowercase   )
															    or config.num_hidden_layers
															)
															__lowercase					=				getattr(lowercase			,     """num_kv_heads"""			,     config.num_attention_heads   )
															__lowercase					=				getattr(lowercase			,     """d_model"""			,     config.hidden_size   )
															__lowercase					=				embed_dim // num_attention_heads

															__lowercase					=				outputs["""past_key_values"""]
															self.assertEqual(len(lowercase   )			,     lowercase   )

															__lowercase	,			__lowercase					=				inputs["""input_ids"""].shape
															for i in range(lowercase   ):
																					if config.new_decoder_architecture:
																											__lowercase					=				config.num_attention_heads
																					elif config.multi_query:
																											__lowercase					=				1
																					self.assertEqual(len(past_kv[0]   )			,     2   )  # K V for the decoder = 2
																					self.assertEqual(
																					    past_kv[i][0].shape			,     (batch_size, num_attention_heads, seq_length, per_head_embed_dim)   )
																					self.assertEqual(
																					    past_kv[i][1].shape			,     (batch_size, num_attention_heads, seq_length, per_head_embed_dim)   )





@require_torch
class 			_lowerCAmelCase   ( unittest.TestCase			):






			"""simple docstring"""





			@slow
			def 		snake_case__			(				self	:       List[Any]   )						->					Dict:
									"""simple docstring"""







									__lowercase					=				AutoTokenizer.from_pretrained("""Rocketknight1/falcon-rw-1b"""   )
									__lowercase					=				FalconForCausalLM.from_pretrained("""Rocketknight1/falcon-rw-1b"""   )
									model.eval()
									model.to(lowercase   )
									__lowercase					=				tokenizer("""My favorite food is"""			,     return_tensors="""pt"""   ).to(lowercase   )

									__lowercase					=				(
									    """My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday."""
									)

									__lowercase					=				model.generate(**lowercase			,     do_sample=lowercase			,     max_new_tokens=19   )
									__lowercase					=				tokenizer.batch_decode(lowercase   )[0]

									self.assertEqual(lowercase			,     lowercase   )





			@slow
			def 		snake_case__			(				self	:       int   )						->					Dict:
									"""simple docstring"""







									for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
															__lowercase					=				AutoTokenizer.from_pretrained(lowercase   )
															__lowercase					=				FalconForCausalLM.from_pretrained(lowercase   )
															model.eval()
															model.to(lowercase   )
															__lowercase					=				tokenizer("""My favorite food is"""			,     return_tensors="""pt"""   ).to(lowercase   )

															# We just test that these run without errors - the models are randomly initialized
															# and so the actual text outputs will be garbage
															model.generate(**lowercase			,     do_sample=lowercase			,     max_new_tokens=4   )
															model.generate(**lowercase			,     do_sample=lowercase			,     max_new_tokens=4   )
															model.generate(**lowercase			,     num_beams=2			,     max_new_tokens=4   )







			@slow
			def 		snake_case__			(				self	:       Tuple   )						->					Dict:
									"""simple docstring"""







									with torch.no_grad():
															for repo in [
															    "Rocketknight1/falcon-rw-1b",
															    "Rocketknight1/tiny-random-falcon-7b",
															    "Rocketknight1/tiny-random-falcon-40b",
															]:
																					__lowercase					=				AutoTokenizer.from_pretrained(lowercase   )
																					__lowercase					=				FalconForCausalLM.from_pretrained(lowercase   )
																					model.eval()
																					model.to(device=lowercase   )
																					__lowercase					=				tokenizer("""My favorite food is"""			,     return_tensors="""pt"""   ).to(lowercase   )

																					# Test results are the same with and without cache
																					__lowercase					=				model.generate(**lowercase			,     do_sample=lowercase			,     max_new_tokens=20			,     use_cache=lowercase   )
																					__lowercase					=				model.generate(**lowercase			,     do_sample=lowercase			,     max_new_tokens=20			,     use_cache=lowercase   )
																					self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0   )










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCamelCase__ =						{
    "configuration_ctrl": ["CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP", "CTRLConfig"],
    "tokenization_ctrl": ["CTRLTokenizer"],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCamelCase__ =						[
			    "CTRL_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "CTRLForSequenceClassification",
			    "CTRLLMHeadModel",
			    "CTRLModel",
			    "CTRLPreTrainedModel",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCamelCase__ =						[
			    "TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFCTRLForSequenceClassification",
			    "TFCTRLLMHeadModel",
			    "TFCTRLModel",
			    "TFCTRLPreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
			from .tokenization_ctrl import CTRLTokenizer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_ctrl import (
						    CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    CTRLForSequenceClassification,
						    CTRLLMHeadModel,
						    CTRLModel,
						    CTRLPreTrainedModel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_ctrl import (
						    TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFCTRLForSequenceClassification,
						    TFCTRLLMHeadModel,
						    TFCTRLModel,
						    TFCTRLPreTrainedModel,
						)

else:
			import sys

			UpperCamelCase__ =						_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





