



import unittest

from transformers import BertGenerationConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import BertGenerationDecoder, BertGenerationEncoder



class 						UpperCAmelCase_					:







							def __init__(     self,    __a,    __a=13,    __a=7,    __a=True,    __a=True,    __a=99,    __a=32,    __a=5,    __a=4,    __a=37,    __a="gelu",    __a=0.1,    __a=0.1,    __a=50,    __a=0.02,    __a=True,    __a=None,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							Optional[int]				     =					parent
														_lowerCAmelCase  :							Optional[int]				     =					batch_size
														_lowerCAmelCase  :							int				     =					seq_length
														_lowerCAmelCase  :							Dict				     =					is_training
														_lowerCAmelCase  :							Tuple				     =					use_input_mask
														_lowerCAmelCase  :							str				     =					vocab_size
														_lowerCAmelCase  :							List[Any]				     =					hidden_size
														_lowerCAmelCase  :							Dict				     =					num_hidden_layers
														_lowerCAmelCase  :							Optional[int]				     =					num_attention_heads
														_lowerCAmelCase  :							int				     =					intermediate_size
														_lowerCAmelCase  :							Union[str, Any]				     =					hidden_act
														_lowerCAmelCase  :							Dict				     =					hidden_dropout_prob
														_lowerCAmelCase  :							Union[str, Any]				     =					attention_probs_dropout_prob
														_lowerCAmelCase  :							Optional[int]				     =					max_position_embeddings
														_lowerCAmelCase  :							Optional[int]				     =					initializer_range
														_lowerCAmelCase  :							int				     =					use_labels
														_lowerCAmelCase  :							Union[str, Any]				     =					scope







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							Any				     =					ids_tensor([self.batch_size, self.seq_length],    self.vocab_size)

														_lowerCAmelCase  :							Dict				     =					None
														if self.use_input_mask:
																					_lowerCAmelCase  :							int				     =					random_attention_mask([self.batch_size, self.seq_length])

														if self.use_labels:
																					_lowerCAmelCase  :							int				     =					ids_tensor([self.batch_size, self.seq_length],    self.vocab_size)

														_lowerCAmelCase  :							Tuple				     =					self.get_config()

														return config, input_ids, input_mask, token_labels







							def      snake_case__   (     self):

														'''simple docstring'''

														return BertGenerationConfig(
														    vocab_size=self.vocab_size,    hidden_size=self.hidden_size,    num_hidden_layers=self.num_hidden_layers,    num_attention_heads=self.num_attention_heads,    intermediate_size=self.intermediate_size,    hidden_act=self.hidden_act,    hidden_dropout_prob=self.hidden_dropout_prob,    attention_probs_dropout_prob=self.attention_probs_dropout_prob,    max_position_embeddings=self.max_position_embeddings,    is_decoder=__a,    initializer_range=self.initializer_range,    )







							def      snake_case__   (     self):

														'''simple docstring'''

														(
														    (
														    _lowerCAmelCase
														)     ,					(
														    _lowerCAmelCase
														)     ,					(
														    _lowerCAmelCase
														)     ,					(
														    _lowerCAmelCase
														)     ,					
														)  :							Optional[int]				     =					self.prepare_config_and_inputs()

														_lowerCAmelCase  :							int				     =					True
														_lowerCAmelCase  :							str				     =					floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
														_lowerCAmelCase  :							Tuple				     =					ids_tensor([self.batch_size, self.seq_length],    vocab_size=2)

														return (
														    config,
														    input_ids,
														    input_mask,
														    token_labels,
														    encoder_hidden_states,
														    encoder_attention_mask,
														)







							def      snake_case__   (     self,    __a,    __a,    __a,    __a,    **__a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							Union[str, Any]				     =					BertGenerationEncoder(config=__a)
														model.to(__a)
														model.eval()
														_lowerCAmelCase  :							Optional[int]				     =					model(__a,    attention_mask=__a)
														_lowerCAmelCase  :							str				     =					model(__a)
														self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.hidden_size))







							def      snake_case__   (     self,    __a,    __a,    __a,    __a,    __a,    __a,    **__a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							Dict				     =					True
														_lowerCAmelCase  :							Dict				     =					BertGenerationEncoder(config=__a)
														model.to(__a)
														model.eval()
														_lowerCAmelCase  :							Dict				     =					model(
														    __a,    attention_mask=__a,    encoder_hidden_states=__a,    encoder_attention_mask=__a,    )
														_lowerCAmelCase  :							List[str]				     =					model(
														    __a,    attention_mask=__a,    encoder_hidden_states=__a,    )
														self.parent.assertEqual(result.last_hidden_state.shape,    (self.batch_size, self.seq_length, self.hidden_size))







							def      snake_case__   (     self,    __a,    __a,    __a,    __a,    __a,    __a,    **__a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							Optional[Any]				     =					True
														_lowerCAmelCase  :							List[Any]				     =					True
														_lowerCAmelCase  :							Dict				     =					BertGenerationDecoder(config=__a).to(__a).eval()

														# first forward pass
														_lowerCAmelCase  :							List[str]				     =					model(
														    __a,    attention_mask=__a,    encoder_hidden_states=__a,    encoder_attention_mask=__a,    use_cache=__a,    )
														_lowerCAmelCase  :							List[Any]				     =					outputs.past_key_values

														# create hypothetical multiple next token and extent to next_input_ids
														_lowerCAmelCase  :							str				     =					ids_tensor((self.batch_size, 3),    config.vocab_size)
														_lowerCAmelCase  :							Tuple				     =					ids_tensor((self.batch_size, 3),    vocab_size=2)

														# append to next input_ids and
														_lowerCAmelCase  :							List[Any]				     =					torch.cat([input_ids, next_tokens],    dim=-1)
														_lowerCAmelCase  :							Union[str, Any]				     =					torch.cat([input_mask, next_mask],    dim=-1)

														_lowerCAmelCase  :							Optional[int]				     =					model(
														    __a,    attention_mask=__a,    encoder_hidden_states=__a,    encoder_attention_mask=__a,    output_hidden_states=__a,    )["hidden_states"][0]
														_lowerCAmelCase  :							Optional[int]				     =					model(
														    __a,    attention_mask=__a,    encoder_hidden_states=__a,    encoder_attention_mask=__a,    past_key_values=__a,    output_hidden_states=__a,    )["hidden_states"][0]

														# select random slice
														_lowerCAmelCase  :							int				     =					ids_tensor((1,),    output_from_past.shape[-1]).item()
														_lowerCAmelCase  :							Tuple				     =					output_from_no_past[:, -3:, random_slice_idx].detach()
														_lowerCAmelCase  :							Optional[Any]				     =					output_from_past[:, :, random_slice_idx].detach()

														self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

														# test that outputs are equal for slice
														self.parent.assertTrue(torch.allclose(__a,    __a,    atol=1E-3))







							def      snake_case__   (     self,    __a,    __a,    __a,    __a,    *__a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							Dict				     =					BertGenerationDecoder(__a)
														model.to(__a)
														model.eval()
														_lowerCAmelCase  :							Any				     =					model(__a,    attention_mask=__a,    labels=__a)
														self.parent.assertEqual(result.logits.shape,    (self.batch_size, self.seq_length, self.vocab_size))







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase  :							str				     =					self.prepare_config_and_inputs()
														_lowerCAmelCase  :							Optional[int]				     =					{"input_ids": input_ids, "attention_mask": input_mask}
														return config, inputs_dict



@require_torch
class 						UpperCAmelCase_					( a     ,      a     ,      a     ,      unittest.TestCase):
							lowerCamelCase__         =	(BertGenerationEncoder, BertGenerationDecoder) if is_torch_available() else ()
							lowerCamelCase__         =	(BertGenerationDecoder,) if is_torch_available() else ()
							lowerCamelCase__         =	(
							    {'feature-extraction': BertGenerationEncoder, 'text-generation': BertGenerationDecoder}
							    if is_torch_available()
							    else {}
							)







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							Any				     =					BertGenerationEncoderTester(self)
														_lowerCAmelCase  :							Union[str, Any]				     =					ConfigTester(self,    config_class=__a,    hidden_size=37)







							def      snake_case__   (     self):

														'''simple docstring'''

														self.config_tester.run_common_tests()







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							Optional[int]				     =					self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*__a)







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase  :							List[Any]				     =					self.model_tester.prepare_config_and_inputs()
														_lowerCAmelCase  :							Optional[int]				     =					"bert"
														self.model_tester.create_and_check_model(__a,    __a,    __a,    __a)







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							Tuple				     =					self.model_tester.prepare_config_and_inputs_for_decoder()
														self.model_tester.create_and_check_model_as_decoder(*__a)







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							Optional[Any]				     =					self.model_tester.prepare_config_and_inputs_for_decoder()
														self.model_tester.create_and_check_decoder_model_past_large_inputs(*__a)







							def      snake_case__   (     self):

														'''simple docstring'''

														(
														    (
														    _lowerCAmelCase
														)     ,					(
														    _lowerCAmelCase
														)     ,					(
														    _lowerCAmelCase
														)     ,					(
														    _lowerCAmelCase
														)     ,					(
														    _lowerCAmelCase
														)     ,					(
														    _lowerCAmelCase
														)     ,					
														)  :							Optional[Any]				     =					self.model_tester.prepare_config_and_inputs_for_decoder()

														_lowerCAmelCase  :							str				     =					None

														self.model_tester.create_and_check_model_as_decoder(
														    __a,    __a,    __a,    __a,    __a,    __a,    )







							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							int				     =					self.model_tester.prepare_config_and_inputs_for_decoder()
														self.model_tester.create_and_check_for_causal_lm(*__a)







							@slow
							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							int				     =					BertGenerationEncoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder")
														self.assertIsNotNone(__a)



@require_torch
class 						UpperCAmelCase_					( unittest.TestCase):







							@slow
							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							Any				     =					BertGenerationEncoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder")
														_lowerCAmelCase  :							Union[str, Any]				     =					torch.tensor([[101, 7592, 1010, 2026, 3899, 2003, 1_0140, 102]])
														with torch.no_grad():
																					_lowerCAmelCase  :							Any				     =					model(__a)[0]
														_lowerCAmelCase  :							Any				     =					torch.Size([1, 8, 1024])
														self.assertEqual(output.shape,    __a)
														_lowerCAmelCase  :							Dict				     =					torch.tensor(
														    [[[0.1_775, 0.0_083, -0.0_321], [1.6_002, 0.1_287, 0.3_912], [2.1_473, 0.5_791, 0.6_066]]])
														self.assertTrue(torch.allclose(output[:, :3, :3],    __a,    atol=1E-4))



@require_torch
class 						UpperCAmelCase_					( unittest.TestCase):







							@slow
							def      snake_case__   (     self):

														'''simple docstring'''

														_lowerCAmelCase  :							Dict				     =					BertGenerationDecoder.from_pretrained("google/bert_for_seq_generation_L-24_bbc_encoder")
														_lowerCAmelCase  :							List[str]				     =					torch.tensor([[101, 7592, 1010, 2026, 3899, 2003, 1_0140, 102]])
														with torch.no_grad():
																					_lowerCAmelCase  :							List[str]				     =					model(__a)[0]
														_lowerCAmelCase  :							str				     =					torch.Size([1, 8, 5_0358])
														self.assertEqual(output.shape,    __a)
														_lowerCAmelCase  :							Optional[Any]				     =					torch.tensor(
														    [[[-0.5_788, -2.5_994, -3.7_054], [0.0_438, 4.7_997, 1.8_795], [1.5_862, 6.6_409, 4.4_638]]])
														self.assertTrue(torch.allclose(output[:, :3, :3],    __a,    atol=1E-4))





import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case	=						logging.get_logger(__name__)

_snake_case	=						{
    "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/config.json",
    "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/config.json",
}



class 						UpperCAmelCase_					( a):
							lowerCamelCase__         =	'xlnet'
							lowerCamelCase__         =	['mems']
							lowerCamelCase__         =	{
							    'n_token': 'vocab_size',  # Backward compatibility
							    'hidden_size': 'd_model',
							    'num_attention_heads': 'n_head',
							    'num_hidden_layers': 'n_layer',
							}







							def __init__(     self,    __a=3_2000,    __a=1024,    __a=24,    __a=16,    __a=4096,    __a="gelu",    __a=True,    __a="bi",    __a=0.02,    __a=1E-12,    __a=0.1,    __a=512,    __a=None,    __a=True,    __a=False,    __a=False,    __a=-1,    __a=False,    __a="last",    __a=True,    __a="tanh",    __a=0.1,    __a=5,    __a=5,    __a=5,    __a=1,    __a=2,    **__a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							int				     =					vocab_size
														_lowerCAmelCase  :							Optional[int]				     =					d_model
														_lowerCAmelCase  :							Tuple				     =					n_layer
														_lowerCAmelCase  :							List[Any]				     =					n_head
														if d_model % n_head != 0:
																					raise ValueError(f"'d_model % n_head' ({d_model % n_head}) should be equal to 0")
														if "d_head" in kwargs:
																					if kwargs["d_head"] != d_model // n_head:
																												raise ValueError(
																												    f"`d_head` ({kwargs['d_head']}) should be equal to `d_model // n_head` ({d_model // n_head})")
														_lowerCAmelCase  :							Optional[int]				     =					d_model // n_head
														_lowerCAmelCase  :							List[str]				     =					ff_activation
														_lowerCAmelCase  :							Tuple				     =					d_inner
														_lowerCAmelCase  :							List[Any]				     =					untie_r
														_lowerCAmelCase  :							List[str]				     =					attn_type

														_lowerCAmelCase  :							Union[str, Any]				     =					initializer_range
														_lowerCAmelCase  :							Any				     =					layer_norm_eps

														_lowerCAmelCase  :							List[Any]				     =					dropout
														_lowerCAmelCase  :							Optional[int]				     =					mem_len
														_lowerCAmelCase  :							Union[str, Any]				     =					reuse_len
														_lowerCAmelCase  :							List[str]				     =					bi_data
														_lowerCAmelCase  :							List[str]				     =					clamp_len
														_lowerCAmelCase  :							Any				     =					same_length

														_lowerCAmelCase  :							List[str]				     =					summary_type
														_lowerCAmelCase  :							int				     =					summary_use_proj
														_lowerCAmelCase  :							Optional[Any]				     =					summary_activation
														_lowerCAmelCase  :							Tuple				     =					summary_last_dropout
														_lowerCAmelCase  :							Union[str, Any]				     =					start_n_top
														_lowerCAmelCase  :							Optional[int]				     =					end_n_top

														_lowerCAmelCase  :							Tuple				     =					bos_token_id
														_lowerCAmelCase  :							List[Any]				     =					pad_token_id
														_lowerCAmelCase  :							Dict				     =					eos_token_id

														if "use_cache" in kwargs:
																					warnings.warn(
																					    "The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`"
																					    " instead.",    __a,    )
																					_lowerCAmelCase  :							Union[str, Any]				     =					kwargs["use_cache"]

														_lowerCAmelCase  :							Union[str, Any]				     =					use_mems_eval
														_lowerCAmelCase  :							Any				     =					use_mems_train
														super().__init__(pad_token_id=__a,    bos_token_id=__a,    eos_token_id=__a,    **__a)







							@property
							def      snake_case__   (     self):

														'''simple docstring'''

														logger.info(f"The model {self.model_type} is one of the few models that has no sequence length limit.")
														return -1







							@max_position_embeddings.setter
							def      snake_case__   (     self,    __a):

														'''simple docstring'''

														raise NotImplementedError(
														    f"The model {self.model_type} is one of the few models that has no sequence length limit.")
