



'''simple docstring'''


import os
import tempfile
import unittest

from transformers import NezhaConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          NezhaForMaskedLM,
          NezhaForMultipleChoice,
          NezhaForNextSentencePrediction,
          NezhaForPreTraining,
          NezhaForQuestionAnswering,
          NezhaForSequenceClassification,
          NezhaForTokenClassification,
          NezhaModel,
      )
      from transformers.models.nezha.modeling_nezha import NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST




class      __UpperCamelCase       :







 def __init__(    self ,				_lowerCAmelCase ,				_lowerCAmelCase=13 ,				_lowerCAmelCase=7 ,				_lowerCAmelCase=True ,				_lowerCAmelCase=True ,				_lowerCAmelCase=True ,				_lowerCAmelCase=True ,				_lowerCAmelCase=99 ,				_lowerCAmelCase=32 ,				_lowerCAmelCase=5 ,				_lowerCAmelCase=4 ,				_lowerCAmelCase=37 ,				_lowerCAmelCase="gelu" ,				_lowerCAmelCase=0.1 ,				_lowerCAmelCase=0.1 ,				_lowerCAmelCase=128 ,				_lowerCAmelCase=32 ,				_lowerCAmelCase=16 ,				_lowerCAmelCase=2 ,				_lowerCAmelCase=0.02 ,				_lowerCAmelCase=3 ,				_lowerCAmelCase=4 ,				_lowerCAmelCase=None ,				)    ->    Optional[Any]:
       '''simple docstring'''






       lowercase        =							parent
       lowercase        =							batch_size
       lowercase        =							seq_length
       lowercase        =							is_training
       lowercase        =							use_input_mask
       lowercase        =							use_token_type_ids
       lowercase        =							use_labels
       lowercase        =							vocab_size
       lowercase        =							hidden_size
       lowercase        =							num_hidden_layers
       lowercase        =							num_attention_heads
       lowercase        =							intermediate_size
       lowercase        =							hidden_act
       lowercase        =							hidden_dropout_prob
       lowercase        =							attention_probs_dropout_prob
       lowercase        =							max_position_embeddings
       lowercase        =							type_vocab_size
       lowercase        =							type_sequence_label_size
       lowercase        =							initializer_range
       lowercase        =							num_labels
       lowercase        =							num_choices
       lowercase        =							scope







 def   _a   (    self						)    ->    Optional[int]:
       '''simple docstring'''






       lowercase        =							ids_tensor([self.batch_size, self.seq_length] ,				self.vocab_size						)

       lowercase        =							None
       if self.use_input_mask:
             lowercase        =							random_attention_mask([self.batch_size, self.seq_length]						)

       lowercase        =							None
       if self.use_token_type_ids:
             lowercase        =							ids_tensor([self.batch_size, self.seq_length] ,				self.type_vocab_size						)

       lowercase        =							None
       lowercase        =							None
       lowercase        =							None
       if self.use_labels:
             lowercase        =							ids_tensor([self.batch_size] ,				self.type_sequence_label_size						)
             lowercase        =							ids_tensor([self.batch_size, self.seq_length] ,				self.num_labels						)
             lowercase        =							ids_tensor([self.batch_size] ,				self.num_choices						)

       lowercase        =							self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







 def   _a   (    self						)    ->    str:
       '''simple docstring'''






       return NezhaConfig(
           vocab_size=self.vocab_size ,				hidden_size=self.hidden_size ,				num_hidden_layers=self.num_hidden_layers ,				num_attention_heads=self.num_attention_heads ,				intermediate_size=self.intermediate_size ,				hidden_act=self.hidden_act ,				hidden_dropout_prob=self.hidden_dropout_prob ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob ,				max_position_embeddings=self.max_position_embeddings ,				type_vocab_size=self.type_vocab_size ,				is_decoder=lowerCAmelCase_ ,				initializer_range=self.initializer_range ,				)







 def   _a   (    self						)    ->    List[Any]:
       '''simple docstring'''






       (
           (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   						
       )        =							self.prepare_config_and_inputs()

       lowercase        =							True
       lowercase        =							floats_tensor([self.batch_size, self.seq_length, self.hidden_size]						)
       lowercase        =							ids_tensor([self.batch_size, self.seq_length] ,				vocab_size=2						)

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase						)    ->    int:
       '''simple docstring'''






       lowercase        =							NezhaModel(config=lowerCAmelCase_						)
       model.to(lowerCAmelCase_						)
       model.eval()
       lowercase        =							model(lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_						)
       lowercase        =							model(lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_						)
       lowercase        =							model(lowerCAmelCase_						)
       self.parent.assertEqual(result.last_hidden_state.shape ,				(self.batch_size, self.seq_length, self.hidden_size)						)
       self.parent.assertEqual(result.pooler_output.shape ,				(self.batch_size, self.hidden_size)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				)    ->    int:
       '''simple docstring'''






       lowercase        =							True
       lowercase        =							NezhaModel(lowerCAmelCase_						)
       model.to(lowerCAmelCase_						)
       model.eval()
       lowercase        =							model(
           lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_ ,				encoder_hidden_states=lowerCAmelCase_ ,				encoder_attention_mask=lowerCAmelCase_ ,				)
       lowercase        =							model(
           lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_ ,				encoder_hidden_states=lowerCAmelCase_ ,				)
       lowercase        =							model(lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_						)
       self.parent.assertEqual(result.last_hidden_state.shape ,				(self.batch_size, self.seq_length, self.hidden_size)						)
       self.parent.assertEqual(result.pooler_output.shape ,				(self.batch_size, self.hidden_size)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase						)    ->    Optional[int]:
       '''simple docstring'''






       lowercase        =							NezhaForMaskedLM(config=lowerCAmelCase_						)
       model.to(lowerCAmelCase_						)
       model.eval()
       lowercase        =							model(lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_ ,				labels=lowerCAmelCase_						)
       self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.seq_length, self.vocab_size)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase						)    ->    Optional[Any]:
       '''simple docstring'''






       lowercase        =							NezhaForNextSentencePrediction(config=lowerCAmelCase_						)
       model.to(lowerCAmelCase_						)
       model.eval()
       lowercase        =							model(
           lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_ ,				labels=lowerCAmelCase_ ,				)
       self.parent.assertEqual(result.logits.shape ,				(self.batch_size, 2)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase						)    ->    Optional[int]:
       '''simple docstring'''






       lowercase        =							NezhaForPreTraining(config=lowerCAmelCase_						)
       model.to(lowerCAmelCase_						)
       model.eval()
       lowercase        =							model(
           lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_ ,				labels=lowerCAmelCase_ ,				next_sentence_label=lowerCAmelCase_ ,				)
       self.parent.assertEqual(result.prediction_logits.shape ,				(self.batch_size, self.seq_length, self.vocab_size)						)
       self.parent.assertEqual(result.seq_relationship_logits.shape ,				(self.batch_size, 2)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase						)    ->    Dict:
       '''simple docstring'''






       lowercase        =							NezhaForQuestionAnswering(config=lowerCAmelCase_						)
       model.to(lowerCAmelCase_						)
       model.eval()
       lowercase        =							model(
           lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_ ,				start_positions=lowerCAmelCase_ ,				end_positions=lowerCAmelCase_ ,				)
       self.parent.assertEqual(result.start_logits.shape ,				(self.batch_size, self.seq_length)						)
       self.parent.assertEqual(result.end_logits.shape ,				(self.batch_size, self.seq_length)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase						)    ->    List[Any]:
       '''simple docstring'''






       lowercase        =							self.num_labels
       lowercase        =							NezhaForSequenceClassification(lowerCAmelCase_						)
       model.to(lowerCAmelCase_						)
       model.eval()
       lowercase        =							model(lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_ ,				labels=lowerCAmelCase_						)
       self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.num_labels)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase						)    ->    Tuple:
       '''simple docstring'''






       lowercase        =							self.num_labels
       lowercase        =							NezhaForTokenClassification(config=lowerCAmelCase_						)
       model.to(lowerCAmelCase_						)
       model.eval()
       lowercase        =							model(lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_ ,				labels=lowerCAmelCase_						)
       self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.seq_length, self.num_labels)						)







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase						)    ->    Dict:
       '''simple docstring'''






       lowercase        =							self.num_choices
       lowercase        =							NezhaForMultipleChoice(config=lowerCAmelCase_						)
       model.to(lowerCAmelCase_						)
       model.eval()
       lowercase        =							input_ids.unsqueeze(1						).expand(-1 ,				self.num_choices ,				-1						).contiguous()
       lowercase        =							token_type_ids.unsqueeze(1						).expand(-1 ,				self.num_choices ,				-1						).contiguous()
       lowercase        =							input_mask.unsqueeze(1						).expand(-1 ,				self.num_choices ,				-1						).contiguous()
       lowercase        =							model(
           lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_ ,				token_type_ids=lowerCAmelCase_ ,				labels=lowerCAmelCase_ ,				)
       self.parent.assertEqual(result.logits.shape ,				(self.batch_size, self.num_choices)						)







 def   _a   (    self						)    ->    int:
       '''simple docstring'''






       lowercase        =							self.prepare_config_and_inputs()
       (
           (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   						
       )        =							config_and_inputs
       lowercase        =							{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
       return config, inputs_dict




@require_torch
class      __UpperCamelCase       (__a			,   __a			,   __a			,   unittest.TestCase			):
 __A			  =					(
     (
         NezhaModel,
         NezhaForMaskedLM,
         NezhaForMultipleChoice,
         NezhaForNextSentencePrediction,
         NezhaForPreTraining,
         NezhaForQuestionAnswering,
         NezhaForSequenceClassification,
         NezhaForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 __A			  =					(
     {
         "feature-extraction": NezhaModel,
         "fill-mask": NezhaForMaskedLM,
         "question-answering": NezhaForQuestionAnswering,
         "text-classification": NezhaForSequenceClassification,
         "token-classification": NezhaForTokenClassification,
         "zero-shot": NezhaForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 __A			  =					True







 def   _a   (    self ,				_lowerCAmelCase ,				_lowerCAmelCase ,				_lowerCAmelCase=False						)    ->    Optional[int]:
       '''simple docstring'''






       lowercase        =							super()._prepare_for_class(lowerCAmelCase_ ,				lowerCAmelCase_ ,				return_labels=lowerCAmelCase_						)

       if return_labels:
             if model_class in get_values(lowerCAmelCase_						):
                   lowercase        =							torch.zeros(
                       (self.model_tester.batch_size, self.model_tester.seq_length) ,				dtype=torch.long ,				device=lowerCAmelCase_						)
                   lowercase        =							torch.zeros(
                       self.model_tester.batch_size ,				dtype=torch.long ,				device=lowerCAmelCase_						)
       return inputs_dict







 def   _a   (    self						)    ->    Optional[Any]:
       '''simple docstring'''






       lowercase        =							NezhaModelTester(self						)
       lowercase        =							ConfigTester(self ,				config_class=lowerCAmelCase_ ,				hidden_size=37						)







 def   _a   (    self						)    ->    Union[str, Any]:
       '''simple docstring'''






       self.config_tester.run_common_tests()







 def   _a   (    self						)    ->    Optional[Any]:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*lowerCAmelCase_						)







 def   _a   (    self						)    ->    Tuple:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_model_as_decoder(*lowerCAmelCase_						)







 def   _a   (    self						)    ->    Dict:
       '''simple docstring'''






       (
           (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   (
           lowercase						
       )     ,   						
       )        =							self.model_tester.prepare_config_and_inputs_for_decoder()

       lowercase        =							None

       self.model_tester.create_and_check_model_as_decoder(
           lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ ,				)







 def   _a   (    self						)    ->    Optional[Any]:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase_						)







 def   _a   (    self						)    ->    List[Any]:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase_						)







 def   _a   (    self						)    ->    str:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_next_sequence_prediction(*lowerCAmelCase_						)







 def   _a   (    self						)    ->    Tuple:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase_						)







 def   _a   (    self						)    ->    int:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase_						)







 def   _a   (    self						)    ->    str:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase_						)







 def   _a   (    self						)    ->    Optional[int]:
       '''simple docstring'''






       lowercase        =							self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase_						)







 @slow
 def   _a   (    self						)    ->    Optional[Any]:
       '''simple docstring'''






       for model_name in NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowercase        =							NezhaModel.from_pretrained(lowerCAmelCase_						)
             self.assertIsNotNone(lowerCAmelCase_						)







 @slow
 @require_torch_gpu
 def   _a   (    self						)    ->    List[str]:
       '''simple docstring'''






       lowercase     ,   lowercase        =							self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes:
             # NezhaForMultipleChoice behaves incorrectly in JIT environments.
             if model_class == NezhaForMultipleChoice:
                   return

             lowercase        =							True
             lowercase        =							model_class(config=lowerCAmelCase_						)

             lowercase        =							self._prepare_for_class(lowerCAmelCase_ ,				lowerCAmelCase_						)
             lowercase        =							torch.jit.trace(
                 lowerCAmelCase_ ,				(inputs_dict["""input_ids"""].to("""cpu"""						), inputs_dict["""attention_mask"""].to("""cpu"""						))						)

             with tempfile.TemporaryDirectory() as tmp:
                   torch.jit.save(lowerCAmelCase_ ,				os.path.join(lowerCAmelCase_ ,				"""bert.pt"""						)						)
                   lowercase        =							torch.jit.load(os.path.join(lowerCAmelCase_ ,				"""bert.pt"""						) ,				map_location=lowerCAmelCase_						)
                   loaded(inputs_dict["""input_ids"""].to(lowerCAmelCase_						) ,				inputs_dict["""attention_mask"""].to(lowerCAmelCase_						)						)




@require_torch
class      __UpperCamelCase       (unittest.TestCase			):







 @slow
 def   _a   (    self						)    ->    str:
       '''simple docstring'''






       lowercase        =							NezhaModel.from_pretrained("""sijunhe/nezha-cn-base"""						)
       lowercase        =							torch.tensor([[0, 1, 2, 3, 4, 5]]						)
       lowercase        =							torch.tensor([[0, 1, 1, 1, 1, 1]]						)
       with torch.no_grad():
             lowercase        =							model(lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_						)[0]
       lowercase        =							torch.Size((1, 6, 768)						)
       self.assertEqual(output.shape ,				lowerCAmelCase_						)
       lowercase        =							torch.tensor([[[0.0685, 0.2441, 0.1102], [0.0600, 0.1906, 0.1349], [0.0221, 0.0819, 0.0586]]]						)

       self.assertTrue(torch.allclose(output[:, 1:4, 1:4] ,				lowerCAmelCase_ ,				atol=1E-4						)						)







 @slow
 def   _a   (    self						)    ->    Tuple:
       '''simple docstring'''






       lowercase        =							NezhaForMaskedLM.from_pretrained("""sijunhe/nezha-cn-base"""						)
       lowercase        =							torch.tensor([[0, 1, 2, 3, 4, 5]]						)
       lowercase        =							torch.tensor([[1, 1, 1, 1, 1, 1]]						)
       with torch.no_grad():
             lowercase        =							model(lowerCAmelCase_ ,				attention_mask=lowerCAmelCase_						)[0]
       lowercase        =							torch.Size((1, 6, 2_1128)						)
       self.assertEqual(output.shape ,				lowerCAmelCase_						)
       lowercase        =							torch.tensor(
           [[-2.7939, -1.7902, -2.2189], [-2.8585, -1.8908, -2.3723], [-2.6499, -1.7750, -2.2558]]						)

       self.assertTrue(torch.allclose(output[:, 1:4, 1:4] ,				lowerCAmelCase_ ,				atol=1E-4						)						)







'''simple docstring'''


def      SCREAMING_SNAKE_CASE				(				):
						lowercase        =							[]
						lowercase        =							1

						while len(lowercase_						) < 1E6:
												constant.append(str(lowercase_						)						)
												i += 1

						lowercase        =							"""""".join(lowercase_						)

						return (
						    int(constant[0]						)
						    * int(constant[9]						)
						    * int(constant[99]						)
						    * int(constant[999]						)
						    * int(constant[9999]						)
						    * int(constant[9_9999]						)
						    * int(constant[99_9999]						)
						)


if __name__ == "__main__":
						print(solution())


