

'''simple docstring'''







import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          MegatronBertForCausalLM,
          MegatronBertForMaskedLM,
          MegatronBertForMultipleChoice,
          MegatronBertForNextSentencePrediction,
          MegatronBertForPreTraining,
          MegatronBertForQuestionAnswering,
          MegatronBertForSequenceClassification,
          MegatronBertForTokenClassification,
          MegatronBertModel,
      )


class 					A__      :



    def __init__(					self		,				UpperCamelCase__		,				UpperCamelCase__=13		,				UpperCamelCase__=7		,				UpperCamelCase__=True		,				UpperCamelCase__=True		,				UpperCamelCase__=True		,				UpperCamelCase__=True		,				UpperCamelCase__=99		,				UpperCamelCase__=64		,				UpperCamelCase__=32		,				UpperCamelCase__=5		,				UpperCamelCase__=4		,				UpperCamelCase__=37		,				UpperCamelCase__="gelu"		,				UpperCamelCase__=0.1		,				UpperCamelCase__=0.1		,				UpperCamelCase__=512		,				UpperCamelCase__=16		,				UpperCamelCase__=2		,				UpperCamelCase__=0.02		,				UpperCamelCase__=3		,				UpperCamelCase__=4		,				UpperCamelCase__=None		,				)						->      str:





           '''simple docstring'''
           A_						=      parent
           A_						=      batch_size
           A_						=      seq_length
           A_						=      is_training
           A_						=      use_input_mask
           A_						=      use_token_type_ids
           A_						=      use_labels
           A_						=      vocab_size
           A_						=      hidden_size
           A_						=      embedding_size
           A_						=      num_hidden_layers
           A_						=      num_attention_heads
           A_						=      intermediate_size
           A_						=      hidden_act
           A_						=      hidden_dropout_prob
           A_						=      attention_probs_dropout_prob
           A_						=      max_position_embeddings
           A_						=      type_vocab_size
           A_						=      type_sequence_label_size
           A_						=      initializer_range
           A_						=      num_labels
           A_						=      num_choices
           A_						=      scope



    def 					snake_case_			(					self       )						->      Optional[int]:





           '''simple docstring'''
           A_						=      ids_tensor([self.batch_size, self.seq_length]		,				self.vocab_size       )

           A_						=      None
           if self.use_input_mask:
                  A_						=      random_attention_mask([self.batch_size, self.seq_length]       )

           A_						=      None
           if self.use_token_type_ids:
                  A_						=      ids_tensor([self.batch_size, self.seq_length]		,				self.type_vocab_size       )

           A_						=      None
           A_						=      None
           A_						=      None
           if self.use_labels:
                  A_						=      ids_tensor([self.batch_size]		,				self.type_sequence_label_size       )
                  A_						=      ids_tensor([self.batch_size, self.seq_length]		,				self.num_labels       )
                  A_						=      ids_tensor([self.batch_size]		,				self.num_choices       )

           A_						=      self.get_config()

           return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



    def 					snake_case_			(					self       )						->      Optional[Any]:





           '''simple docstring'''
           return MegatronBertConfig(
               vocab_size=self.vocab_size		,				hidden_size=self.hidden_size		,				num_hidden_layers=self.num_hidden_layers		,				num_attention_heads=self.num_attention_heads		,				intermediate_size=self.intermediate_size		,				embedding_size=self.embedding_size		,				hidden_act=self.hidden_act		,				hidden_dropout_prob=self.hidden_dropout_prob		,				attention_probs_dropout_prob=self.attention_probs_dropout_prob		,				max_position_embeddings=self.max_position_embeddings		,				type_vocab_size=self.type_vocab_size		,				is_decoder=__A		,				initializer_range=self.initializer_range		,				)



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           A_						=      MegatronBertModel(config=__A       )
           model.to(__A       )
           model.eval()
           A_						=      model(__A		,				attention_mask=__A		,				token_type_ids=__A       )
           A_						=      model(__A		,				token_type_ids=__A       )
           A_						=      model(__A       )

           self.parent.assertEqual(result.last_hidden_state.shape		,				(self.batch_size, self.seq_length, self.hidden_size)       )
           self.parent.assertEqual(result.pooler_output.shape		,				(self.batch_size, self.hidden_size)       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      Optional[int]:





           '''simple docstring'''
           A_						=      MegatronBertForMaskedLM(config=__A       )
           model.to(__A       )
           model.eval()
           A_						=      model(__A		,				attention_mask=__A		,				token_type_ids=__A		,				labels=__A       )
           self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.vocab_size)       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      int:





           '''simple docstring'''
           A_						=      MegatronBertForCausalLM(config=__A       )
           model.to(__A       )
           model.eval()
           A_						=      model(__A		,				attention_mask=__A		,				token_type_ids=__A		,				labels=__A       )
           self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.vocab_size)       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      str:





           '''simple docstring'''
           A_						=      MegatronBertForNextSentencePrediction(config=__A       )
           model.to(__A       )
           model.eval()
           A_						=      model(
               __A		,				attention_mask=__A		,				token_type_ids=__A		,				labels=__A		,				)
           self.parent.assertEqual(result.logits.shape		,				(self.batch_size, 2)       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      List[str]:





           '''simple docstring'''
           A_						=      MegatronBertForPreTraining(config=__A       )
           model.to(__A       )
           model.eval()
           A_						=      model(
               __A		,				attention_mask=__A		,				token_type_ids=__A		,				labels=__A		,				next_sentence_label=__A		,				)
           self.parent.assertEqual(result.prediction_logits.shape		,				(self.batch_size, self.seq_length, self.vocab_size)       )
           self.parent.assertEqual(result.seq_relationship_logits.shape		,				(self.batch_size, 2)       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           A_						=      MegatronBertForQuestionAnswering(config=__A       )
           model.to(__A       )
           model.eval()
           A_						=      model(
               __A		,				attention_mask=__A		,				token_type_ids=__A		,				start_positions=__A		,				end_positions=__A		,				)
           self.parent.assertEqual(result.start_logits.shape		,				(self.batch_size, self.seq_length)       )
           self.parent.assertEqual(result.end_logits.shape		,				(self.batch_size, self.seq_length)       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           A_						=      self.num_labels
           A_						=      MegatronBertForSequenceClassification(__A       )
           model.to(__A       )
           model.eval()
           A_						=      model(__A		,				attention_mask=__A		,				token_type_ids=__A		,				labels=__A       )
           self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_labels)       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           A_						=      self.num_labels
           A_						=      MegatronBertForTokenClassification(config=__A       )
           model.to(__A       )
           model.eval()
           A_						=      model(__A		,				attention_mask=__A		,				token_type_ids=__A		,				labels=__A       )
           self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.num_labels)       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__       )						->      Dict:





           '''simple docstring'''
           A_						=      self.num_choices
           A_						=      MegatronBertForMultipleChoice(config=__A       )
           model.to(__A       )
           model.eval()
           A_						=      input_ids.unsqueeze(1       ).expand(-1		,				self.num_choices		,				-1       ).contiguous()
           A_						=      token_type_ids.unsqueeze(1       ).expand(-1		,				self.num_choices		,				-1       ).contiguous()
           A_						=      input_mask.unsqueeze(1       ).expand(-1		,				self.num_choices		,				-1       ).contiguous()
           A_						=      model(
               __A		,				attention_mask=__A		,				token_type_ids=__A		,				labels=__A		,				)
           self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_choices)       )







    def 					snake_case_			(					self       )						->      Any:





           '''simple docstring'''
           A_						=      self.prepare_config_and_inputs()
           (
               (
               A_
           )					,			(
               A_
           )					,			(
               A_
           )					,			(
               A_
           )					,			(
               A_
           )					,			(
               A_
           )					,			(
               A_
           )					,			
           )						=      config_and_inputs
           A_						=      {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
           return config, inputs_dict


@require_torch
class 					A__      (      _snake_case						,						_snake_case						,						unittest.TestCase			):
    lowercase						=		(
        (
            MegatronBertModel,
            MegatronBertForMaskedLM,
            MegatronBertForCausalLM,
            MegatronBertForMultipleChoice,
            MegatronBertForNextSentencePrediction,
            MegatronBertForPreTraining,
            MegatronBertForQuestionAnswering,
            MegatronBertForSequenceClassification,
            MegatronBertForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    lowercase						=		(
        {
            "feature-extraction": MegatronBertModel,
            "fill-mask": MegatronBertForMaskedLM,
            "question-answering": MegatronBertForQuestionAnswering,
            "text-classification": MegatronBertForSequenceClassification,
            "text-generation": MegatronBertForCausalLM,
            "token-classification": MegatronBertForTokenClassification,
            "zero-shot": MegatronBertForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    lowercase						=		True
    # test_resize_embeddings = False
    lowercase						=		False



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__		,				UpperCamelCase__=False       )						->      Dict:





           '''simple docstring'''
           A_						=      super()._prepare_for_class(__A		,				__A		,				return_labels=__A       )

           if return_labels:
                  if model_class in get_values(__A       ):
                         A_						=      torch.zeros(
                             (self.model_tester.batch_size, self.model_tester.seq_length)		,				dtype=torch.long		,				device=__A       )
                         A_						=      torch.zeros(
                             self.model_tester.batch_size		,				dtype=torch.long		,				device=__A       )
           return inputs_dict



    def 					snake_case_			(					self       )						->      str:





           '''simple docstring'''
           A_						=      MegatronBertModelTester(self       )
           A_						=      ConfigTester(self		,				config_class=__A		,				hidden_size=37       )



    def 					snake_case_			(					self       )						->      Any:





           '''simple docstring'''
           self.config_tester.run_common_tests()



    def 					snake_case_			(					self       )						->      List[Any]:





           '''simple docstring'''
           A_						=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_megatron_bert_model(*__A       )



    def 					snake_case_			(					self       )						->      Union[str, Any]:





           '''simple docstring'''
           A_						=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_megatron_bert_for_masked_lm(*__A       )



    def 					snake_case_			(					self       )						->      Optional[Any]:





           '''simple docstring'''
           A_						=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*__A       )



    def 					snake_case_			(					self       )						->      Optional[Any]:





           '''simple docstring'''
           A_						=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*__A       )



    def 					snake_case_			(					self       )						->      Optional[int]:





           '''simple docstring'''
           A_						=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_megatron_bert_for_pretraining(*__A       )



    def 					snake_case_			(					self       )						->      List[str]:





           '''simple docstring'''
           A_						=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_megatron_bert_for_question_answering(*__A       )



    def 					snake_case_			(					self       )						->      Optional[int]:





           '''simple docstring'''
           A_						=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*__A       )







    def 					snake_case_			(					self       )						->      Dict:





           '''simple docstring'''
           A_						=      self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_megatron_bert_for_token_classification(*__A       )

def       UpperCAmelCase__    (      UpperCAmelCase__				)			->			Dict:
       return torch.tensor(
           _lowercase,       dtype=torch.long,       device=_lowercase,       )


__lowerCamelCase		       =    1e-4







@require_torch
@require_sentencepiece
@require_tokenizers
class 					A__      (      unittest.TestCase			):



    @slow
    @unittest.skip("""Model is not available."""       )
    def 					snake_case_			(					self       )						->      Dict:





           '''simple docstring'''
           A_						=      """nvidia/megatron-bert-uncased-345m"""
           if "MYDIR" in os.environ:
                  A_						=      os.path.join(os.environ["""MYDIR"""]		,				__A       )
           A_						=      MegatronBertModel.from_pretrained(__A       )
           model.to(__A       )
           model.half()
           A_						=      _long_tensor([[101, 7110, 1005, 1056, 2023, 11333, 17413, 1029, 102]]       )
           with torch.no_grad():
                  A_						=      model(__A       )[0]
           A_						=      torch.Size((1, 9, 1024)       )
           self.assertEqual(output.shape		,				__A       )

           A_						=      [-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
           for ii in range(3       ):
                  for jj in range(3       ):
                         A_						=      output[0, ii, jj]
                         A_						=      expected[3 * ii + jj]
                         A_						=      """ii={} jj={} a={} b={}""".format(__A		,				__A		,				__A		,				__A       )
                         self.assertTrue(math.isclose(__A		,				__A		,				rel_tol=__A		,				abs_tol=__A       )		,				msg=__A       )



'''simple docstring'''







def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__				)			->			float:
       _validate_point(UpperCAmelCase__				)
       _validate_point(UpperCAmelCase__				)
       if len(UpperCAmelCase__				) != len(UpperCAmelCase__				):
              raise ValueError("""Both points must be in the same n-dimensional space"""				)

       return float(sum(abs(a - b				) for a, b in zip(UpperCAmelCase__,       UpperCAmelCase__				)				)				)
def       UpperCAmelCase__    (      UpperCAmelCase__				)			->			None:
       if point:
              if isinstance(UpperCAmelCase__,       UpperCAmelCase__				):
                     for item in point:
                            if not isinstance(UpperCAmelCase__,       (int, float)				):
                                   A_						=      (
                                       """Expected a list of numbers as input, found """
                                       F'''{type(UpperCAmelCase__				).__name__}'''
                                   )
                                   raise TypeError(UpperCAmelCase__				)
              else:
                     A_						=      F'''Expected a list of numbers as input, found {type(UpperCAmelCase__				).__name__}'''
                     raise TypeError(UpperCAmelCase__				)
       else:
              raise ValueError("""Missing an input"""				)
def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__				)			->			float:
       _validate_point(UpperCAmelCase__				)
       _validate_point(UpperCAmelCase__				)
       if len(UpperCAmelCase__				) != len(UpperCAmelCase__				):
              raise ValueError("""Both points must be in the same n-dimensional space"""				)

       return float(sum(abs(x - y				) for x, y in zip(UpperCAmelCase__,       UpperCAmelCase__				)				)				)


if __name__ == "__main__":
      import doctest

      doctest.testmod()
