






'''simple docstring'''
from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
  import PIL

a_					= logging.get_logger(__name__)





def  _a(  UpperCamelCase__				:						List[Any]							):




      '''simple docstring'''






      if isinstance(UpperCamelCase__,	(list, tuple)							) and isinstance(videos[0],	(list, tuple)							) and is_valid_image(videos[0][0]							):
            return videos

      elif isinstance(UpperCamelCase__,	(list, tuple)							) and is_valid_image(videos[0]							):
            return [videos]

      elif is_valid_image(UpperCamelCase__							):
            return [[videos]]

      raise ValueError(f"Could not make batched video from {videos}"							)







class      __SCREAMING_SNAKE_CASE      (			lowerCamelCase				):
 snake_case_  							=							["""pixel_values"""]




 def __init__(      self	:							int					,      __lowercase	:							bool = True					,      __lowercase	:							Dict[str, int] = None					,      __lowercase	:							PILImageResampling = PILImageResampling.BILINEAR					,      __lowercase	:							bool = True					,      __lowercase	:							Dict[str, int] = None					,      __lowercase	:							bool = True					,      __lowercase	:							Union[int, float] = 1 / 2_55					,      __lowercase	:							bool = True					,      __lowercase	:							bool = True					,      __lowercase	:							Optional[Union[float, List[float]]] = None					,      __lowercase	:							Optional[Union[float, List[float]]] = None					,      **__lowercase	:							Optional[int]					,      )	->						None:
       super().__init__(**__lowercase      )
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=size if size is not None else {'''shortest_edge''': 2_56}
       SCREAMING_SNAKE_CASE__     :				str									=get_size_dict(__lowercase					,      default_to_square=__lowercase      )
       SCREAMING_SNAKE_CASE__     :				str									=crop_size if crop_size is not None else {'''height''': 2_24, '''width''': 2_24}
       SCREAMING_SNAKE_CASE__     :				str									=get_size_dict(__lowercase					,      param_name='''crop_size'''      )

       SCREAMING_SNAKE_CASE__     :				int									=do_resize
       SCREAMING_SNAKE_CASE__     :				Tuple									=size
       SCREAMING_SNAKE_CASE__     :				int									=do_center_crop
       SCREAMING_SNAKE_CASE__     :				Dict									=crop_size
       SCREAMING_SNAKE_CASE__     :				Tuple									=resample
       SCREAMING_SNAKE_CASE__     :				List[Any]									=do_rescale
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=rescale_factor
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=offset
       SCREAMING_SNAKE_CASE__     :				Tuple									=do_normalize
       SCREAMING_SNAKE_CASE__     :				List[str]									=image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
       SCREAMING_SNAKE_CASE__     :				List[Any]									=image_std if image_std is not None else IMAGENET_STANDARD_STD




 def      __magic_name__	(      self	:							Optional[Any]					,      __lowercase	:							np.ndarray					,      __lowercase	:							Dict[str, int]					,      __lowercase	:							PILImageResampling = PILImageResampling.BILINEAR					,      __lowercase	:							Optional[Union[str, ChannelDimension]] = None					,      **__lowercase	:							str					,      )	->						np.ndarray:
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=get_size_dict(__lowercase					,      default_to_square=__lowercase      )
       if "shortest_edge" in size:
             SCREAMING_SNAKE_CASE__     :				Optional[int]									=get_resize_output_image_size(__lowercase					,      size['''shortest_edge''']					,      default_to_square=__lowercase      )
       elif "height" in size and "width" in size:
             SCREAMING_SNAKE_CASE__     :				Any									=(size['''height'''], size['''width'''])
       else:
             raise ValueError(F"Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"      )
       return resize(__lowercase					,      size=__lowercase					,      resample=__lowercase					,      data_format=__lowercase					,      **__lowercase      )




 def      __magic_name__	(      self	:							Optional[Any]					,      __lowercase	:							np.ndarray					,      __lowercase	:							Dict[str, int]					,      __lowercase	:							Optional[Union[str, ChannelDimension]] = None					,      **__lowercase	:							List[Any]					,      )	->						np.ndarray:
       SCREAMING_SNAKE_CASE__     :				Any									=get_size_dict(__lowercase      )
       if "height" not in size or "width" not in size:
             raise ValueError(F"Size must have 'height' and 'width' as keys. Got {size.keys()}"      )
       return center_crop(__lowercase					,      size=(size['''height'''], size['''width'''])					,      data_format=__lowercase					,      **__lowercase      )




 def      __magic_name__	(      self	:							Tuple					,      __lowercase	:							np.ndarray					,      __lowercase	:							Union[int, float]					,      __lowercase	:							bool = True					,      __lowercase	:							Optional[Union[str, ChannelDimension]] = None					,      **__lowercase	:							Optional[int]					,      )	->						List[Any]:
       SCREAMING_SNAKE_CASE__     :				str									=image.astype(np.floataa      )
       if offset:
             SCREAMING_SNAKE_CASE__     :				int									=image - (scale / 2)
       return rescale(__lowercase					,      scale=__lowercase					,      data_format=__lowercase					,      **__lowercase      )




 def      __magic_name__	(      self	:							Optional[Any]					,      __lowercase	:							np.ndarray					,      __lowercase	:							Union[float, List[float]]					,      __lowercase	:							Union[float, List[float]]					,      __lowercase	:							Optional[Union[str, ChannelDimension]] = None					,      **__lowercase	:							Any					,      )	->						np.ndarray:
       return normalize(__lowercase					,      mean=__lowercase					,      std=__lowercase					,      data_format=__lowercase					,      **__lowercase      )




 def      __magic_name__	(      self	:							Dict					,      __lowercase	:							ImageInput					,      __lowercase	:							bool = None					,      __lowercase	:							Dict[str, int] = None					,      __lowercase	:							PILImageResampling = None					,      __lowercase	:							bool = None					,      __lowercase	:							Dict[str, int] = None					,      __lowercase	:							bool = None					,      __lowercase	:							float = None					,      __lowercase	:							bool = None					,      __lowercase	:							bool = None					,      __lowercase	:							Optional[Union[float, List[float]]] = None					,      __lowercase	:							Optional[Union[float, List[float]]] = None					,      __lowercase	:							Optional[ChannelDimension] = ChannelDimension.FIRST					,      )	->						np.ndarray:
       if do_resize and size is None or resample is None:
             raise ValueError('''Size and resample must be specified if do_resize is True.'''      )

       if do_center_crop and crop_size is None:
             raise ValueError('''Crop size must be specified if do_center_crop is True.'''      )

       if do_rescale and rescale_factor is None:
             raise ValueError('''Rescale factor must be specified if do_rescale is True.'''      )

       if do_normalize and (image_mean is None or image_std is None):
             raise ValueError('''Image mean and std must be specified if do_normalize is True.'''      )

       if offset and not do_rescale:
             raise ValueError('''For offset, do_rescale must also be set to True.'''      )

       # All transformations expect numpy arrays.
       SCREAMING_SNAKE_CASE__     :				List[str]									=to_numpy_array(__lowercase      )

       if do_resize:
             SCREAMING_SNAKE_CASE__     :				str									=self.resize(image=__lowercase					,      size=__lowercase					,      resample=__lowercase      )

       if do_center_crop:
             SCREAMING_SNAKE_CASE__     :				Optional[Any]									=self.center_crop(__lowercase					,      size=__lowercase      )

       if do_rescale:
             SCREAMING_SNAKE_CASE__     :				List[str]									=self.rescale(image=__lowercase					,      scale=__lowercase					,      offset=__lowercase      )

       if do_normalize:
             SCREAMING_SNAKE_CASE__     :				int									=self.normalize(image=__lowercase					,      mean=__lowercase					,      std=__lowercase      )

       SCREAMING_SNAKE_CASE__     :				Dict									=to_channel_dimension_format(__lowercase					,      __lowercase      )
       return image




 def      __magic_name__	(      self	:							Any					,      __lowercase	:							ImageInput					,      __lowercase	:							bool = None					,      __lowercase	:							Dict[str, int] = None					,      __lowercase	:							PILImageResampling = None					,      __lowercase	:							bool = None					,      __lowercase	:							Dict[str, int] = None					,      __lowercase	:							bool = None					,      __lowercase	:							float = None					,      __lowercase	:							bool = None					,      __lowercase	:							bool = None					,      __lowercase	:							Optional[Union[float, List[float]]] = None					,      __lowercase	:							Optional[Union[float, List[float]]] = None					,      __lowercase	:							Optional[Union[str, TensorType]] = None					,      __lowercase	:							ChannelDimension = ChannelDimension.FIRST					,      **__lowercase	:							Dict					,      )	->						PIL.Image.Image:
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=do_resize if do_resize is not None else self.do_resize
       SCREAMING_SNAKE_CASE__     :				List[str]									=resample if resample is not None else self.resample
       SCREAMING_SNAKE_CASE__     :				str									=do_center_crop if do_center_crop is not None else self.do_center_crop
       SCREAMING_SNAKE_CASE__     :				Dict									=do_rescale if do_rescale is not None else self.do_rescale
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=rescale_factor if rescale_factor is not None else self.rescale_factor
       SCREAMING_SNAKE_CASE__     :				Any									=offset if offset is not None else self.offset
       SCREAMING_SNAKE_CASE__     :				List[Any]									=do_normalize if do_normalize is not None else self.do_normalize
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=image_mean if image_mean is not None else self.image_mean
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=image_std if image_std is not None else self.image_std

       SCREAMING_SNAKE_CASE__     :				Optional[int]									=size if size is not None else self.size
       SCREAMING_SNAKE_CASE__     :				str									=get_size_dict(__lowercase					,      default_to_square=__lowercase      )
       SCREAMING_SNAKE_CASE__     :				str									=crop_size if crop_size is not None else self.crop_size
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=get_size_dict(__lowercase					,      param_name='''crop_size'''      )

       if not valid_images(__lowercase      ):
             raise ValueError(
                 '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
                 '''torch.Tensor, tf.Tensor or jax.ndarray.'''      )

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=make_batched(__lowercase      )

       SCREAMING_SNAKE_CASE__     :				int									=[
           [
               self._preprocess_image(
                   image=__lowercase					,      do_resize=__lowercase					,      size=__lowercase					,      resample=__lowercase					,      do_center_crop=__lowercase					,      crop_size=__lowercase					,      do_rescale=__lowercase					,      rescale_factor=__lowercase					,      offset=__lowercase					,      do_normalize=__lowercase					,      image_mean=__lowercase					,      image_std=__lowercase					,      data_format=__lowercase					,      )
               for img in video
           ]
           for video in videos
       ]

       SCREAMING_SNAKE_CASE__     :				Tuple									={'''pixel_values''': videos}
       return BatchFeature(data=__lowercase					,      tensor_type=__lowercase      )







'''simple docstring'''
import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      MegatronBertForCausalLM,
      MegatronBertForMaskedLM,
      MegatronBertForMultipleChoice,
      MegatronBertForNextSentencePrediction,
      MegatronBertForPreTraining,
      MegatronBertForQuestionAnswering,
      MegatronBertForSequenceClassification,
      MegatronBertForTokenClassification,
      MegatronBertModel,
  )






class      __SCREAMING_SNAKE_CASE      :




 def __init__(      self	:							Optional[int]					,      __lowercase	:							Optional[Any]					,      __lowercase	:							List[str]=13					,      __lowercase	:							int=7					,      __lowercase	:							Tuple=True					,      __lowercase	:							Optional[Any]=True					,      __lowercase	:							List[Any]=True					,      __lowercase	:							str=True					,      __lowercase	:							Tuple=99					,      __lowercase	:							Union[str, Any]=64					,      __lowercase	:							Tuple=32					,      __lowercase	:							Optional[Any]=5					,      __lowercase	:							Tuple=4					,      __lowercase	:							Optional[int]=37					,      __lowercase	:							int="gelu"					,      __lowercase	:							Union[str, Any]=0.1					,      __lowercase	:							List[Any]=0.1					,      __lowercase	:							List[Any]=5_12					,      __lowercase	:							int=16					,      __lowercase	:							Optional[int]=2					,      __lowercase	:							Tuple=0.02					,      __lowercase	:							List[str]=3					,      __lowercase	:							List[str]=4					,      __lowercase	:							List[str]=None					,      )	->						Tuple:
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=parent
       SCREAMING_SNAKE_CASE__     :				Any									=batch_size
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=seq_length
       SCREAMING_SNAKE_CASE__     :				Dict									=is_training
       SCREAMING_SNAKE_CASE__     :				List[Any]									=use_input_mask
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=use_token_type_ids
       SCREAMING_SNAKE_CASE__     :				List[Any]									=use_labels
       SCREAMING_SNAKE_CASE__     :				int									=vocab_size
       SCREAMING_SNAKE_CASE__     :				str									=hidden_size
       SCREAMING_SNAKE_CASE__     :				Any									=embedding_size
       SCREAMING_SNAKE_CASE__     :				Tuple									=num_hidden_layers
       SCREAMING_SNAKE_CASE__     :				str									=num_attention_heads
       SCREAMING_SNAKE_CASE__     :				Tuple									=intermediate_size
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=hidden_act
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=hidden_dropout_prob
       SCREAMING_SNAKE_CASE__     :				str									=attention_probs_dropout_prob
       SCREAMING_SNAKE_CASE__     :				Any									=max_position_embeddings
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=type_vocab_size
       SCREAMING_SNAKE_CASE__     :				Any									=type_sequence_label_size
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=initializer_range
       SCREAMING_SNAKE_CASE__     :				str									=num_labels
       SCREAMING_SNAKE_CASE__     :				List[str]									=num_choices
       SCREAMING_SNAKE_CASE__     :				List[str]									=scope




 def      __magic_name__	(      self	:							str      )	->						Optional[Any]:
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=ids_tensor([self.batch_size, self.seq_length]					,      self.vocab_size      )

       SCREAMING_SNAKE_CASE__     :				int									=None
       if self.use_input_mask:
             SCREAMING_SNAKE_CASE__     :				List[str]									=random_attention_mask([self.batch_size, self.seq_length]      )

       SCREAMING_SNAKE_CASE__     :				int									=None
       if self.use_token_type_ids:
             SCREAMING_SNAKE_CASE__     :				Any									=ids_tensor([self.batch_size, self.seq_length]					,      self.type_vocab_size      )

       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=None
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=None
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=None
       if self.use_labels:
             SCREAMING_SNAKE_CASE__     :				Tuple									=ids_tensor([self.batch_size]					,      self.type_sequence_label_size      )
             SCREAMING_SNAKE_CASE__     :				List[Any]									=ids_tensor([self.batch_size, self.seq_length]					,      self.num_labels      )
             SCREAMING_SNAKE_CASE__     :				Any									=ids_tensor([self.batch_size]					,      self.num_choices      )

       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




 def      __magic_name__	(      self	:							List[str]      )	->						Any:
       return MegatronBertConfig(
           vocab_size=self.vocab_size					,      hidden_size=self.hidden_size					,      num_hidden_layers=self.num_hidden_layers					,      num_attention_heads=self.num_attention_heads					,      intermediate_size=self.intermediate_size					,      embedding_size=self.embedding_size					,      hidden_act=self.hidden_act					,      hidden_dropout_prob=self.hidden_dropout_prob					,      attention_probs_dropout_prob=self.attention_probs_dropout_prob					,      max_position_embeddings=self.max_position_embeddings					,      type_vocab_size=self.type_vocab_size					,      is_decoder=__lowercase					,      initializer_range=self.initializer_range					,      )




 def      __magic_name__	(      self	:							Any					,      __lowercase	:							Tuple					,      __lowercase	:							Any					,      __lowercase	:							List[Any]					,      __lowercase	:							Optional[int]					,      __lowercase	:							List[str]					,      __lowercase	:							Dict					,      __lowercase	:							Union[str, Any]      )	->						int:
       SCREAMING_SNAKE_CASE__     :				List[str]									=MegatronBertModel(config=__lowercase      )
       model.to(__lowercase      )
       model.eval()
       SCREAMING_SNAKE_CASE__     :				List[Any]									=model(__lowercase					,      attention_mask=__lowercase					,      token_type_ids=__lowercase      )
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=model(__lowercase					,      token_type_ids=__lowercase      )
       SCREAMING_SNAKE_CASE__     :				str									=model(__lowercase      )

       self.parent.assertEqual(result.last_hidden_state.shape					,      (self.batch_size, self.seq_length, self.hidden_size)      )
       self.parent.assertEqual(result.pooler_output.shape					,      (self.batch_size, self.hidden_size)      )




 def      __magic_name__	(      self	:							Dict					,      __lowercase	:							Optional[int]					,      __lowercase	:							List[str]					,      __lowercase	:							str					,      __lowercase	:							Tuple					,      __lowercase	:							Any					,      __lowercase	:							Optional[int]					,      __lowercase	:							Any      )	->						Optional[int]:
       SCREAMING_SNAKE_CASE__     :				str									=MegatronBertForMaskedLM(config=__lowercase      )
       model.to(__lowercase      )
       model.eval()
       SCREAMING_SNAKE_CASE__     :				Tuple									=model(__lowercase					,      attention_mask=__lowercase					,      token_type_ids=__lowercase					,      labels=__lowercase      )
       self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.seq_length, self.vocab_size)      )




 def      __magic_name__	(      self	:							List[str]					,      __lowercase	:							Tuple					,      __lowercase	:							Tuple					,      __lowercase	:							Optional[Any]					,      __lowercase	:							Any					,      __lowercase	:							str					,      __lowercase	:							Any					,      __lowercase	:							str      )	->						Tuple:
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=MegatronBertForCausalLM(config=__lowercase      )
       model.to(__lowercase      )
       model.eval()
       SCREAMING_SNAKE_CASE__     :				Dict									=model(__lowercase					,      attention_mask=__lowercase					,      token_type_ids=__lowercase					,      labels=__lowercase      )
       self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.seq_length, self.vocab_size)      )




 def      __magic_name__	(      self	:							Union[str, Any]					,      __lowercase	:							Optional[int]					,      __lowercase	:							Dict					,      __lowercase	:							Union[str, Any]					,      __lowercase	:							Dict					,      __lowercase	:							Union[str, Any]					,      __lowercase	:							int					,      __lowercase	:							Any      )	->						Dict:
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=MegatronBertForNextSentencePrediction(config=__lowercase      )
       model.to(__lowercase      )
       model.eval()
       SCREAMING_SNAKE_CASE__     :				List[str]									=model(
           __lowercase					,      attention_mask=__lowercase					,      token_type_ids=__lowercase					,      labels=__lowercase					,      )
       self.parent.assertEqual(result.logits.shape					,      (self.batch_size, 2)      )




 def      __magic_name__	(      self	:							List[str]					,      __lowercase	:							Optional[int]					,      __lowercase	:							str					,      __lowercase	:							Optional[int]					,      __lowercase	:							Optional[int]					,      __lowercase	:							Optional[Any]					,      __lowercase	:							str					,      __lowercase	:							Optional[Any]      )	->						Union[str, Any]:
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=MegatronBertForPreTraining(config=__lowercase      )
       model.to(__lowercase      )
       model.eval()
       SCREAMING_SNAKE_CASE__     :				List[str]									=model(
           __lowercase					,      attention_mask=__lowercase					,      token_type_ids=__lowercase					,      labels=__lowercase					,      next_sentence_label=__lowercase					,      )
       self.parent.assertEqual(result.prediction_logits.shape					,      (self.batch_size, self.seq_length, self.vocab_size)      )
       self.parent.assertEqual(result.seq_relationship_logits.shape					,      (self.batch_size, 2)      )




 def      __magic_name__	(      self	:							Optional[int]					,      __lowercase	:							Tuple					,      __lowercase	:							Optional[int]					,      __lowercase	:							Union[str, Any]					,      __lowercase	:							int					,      __lowercase	:							Any					,      __lowercase	:							Any					,      __lowercase	:							List[str]      )	->						Optional[int]:
       SCREAMING_SNAKE_CASE__     :				List[Any]									=MegatronBertForQuestionAnswering(config=__lowercase      )
       model.to(__lowercase      )
       model.eval()
       SCREAMING_SNAKE_CASE__     :				Dict									=model(
           __lowercase					,      attention_mask=__lowercase					,      token_type_ids=__lowercase					,      start_positions=__lowercase					,      end_positions=__lowercase					,      )
       self.parent.assertEqual(result.start_logits.shape					,      (self.batch_size, self.seq_length)      )
       self.parent.assertEqual(result.end_logits.shape					,      (self.batch_size, self.seq_length)      )




 def      __magic_name__	(      self	:							Optional[int]					,      __lowercase	:							Optional[int]					,      __lowercase	:							Optional[int]					,      __lowercase	:							List[Any]					,      __lowercase	:							List[Any]					,      __lowercase	:							Optional[int]					,      __lowercase	:							int					,      __lowercase	:							List[str]      )	->						List[Any]:
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=self.num_labels
       SCREAMING_SNAKE_CASE__     :				Dict									=MegatronBertForSequenceClassification(__lowercase      )
       model.to(__lowercase      )
       model.eval()
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=model(__lowercase					,      attention_mask=__lowercase					,      token_type_ids=__lowercase					,      labels=__lowercase      )
       self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.num_labels)      )




 def      __magic_name__	(      self	:							Optional[int]					,      __lowercase	:							List[Any]					,      __lowercase	:							str					,      __lowercase	:							Optional[Any]					,      __lowercase	:							Any					,      __lowercase	:							Tuple					,      __lowercase	:							Optional[Any]					,      __lowercase	:							int      )	->						List[Any]:
       SCREAMING_SNAKE_CASE__     :				Tuple									=self.num_labels
       SCREAMING_SNAKE_CASE__     :				int									=MegatronBertForTokenClassification(config=__lowercase      )
       model.to(__lowercase      )
       model.eval()
       SCREAMING_SNAKE_CASE__     :				List[str]									=model(__lowercase					,      attention_mask=__lowercase					,      token_type_ids=__lowercase					,      labels=__lowercase      )
       self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.seq_length, self.num_labels)      )




 def      __magic_name__	(      self	:							Dict					,      __lowercase	:							List[str]					,      __lowercase	:							Optional[int]					,      __lowercase	:							Dict					,      __lowercase	:							Any					,      __lowercase	:							List[Any]					,      __lowercase	:							int					,      __lowercase	:							Optional[int]      )	->						Tuple:
       SCREAMING_SNAKE_CASE__     :				int									=self.num_choices
       SCREAMING_SNAKE_CASE__     :				List[str]									=MegatronBertForMultipleChoice(config=__lowercase      )
       model.to(__lowercase      )
       model.eval()
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=input_ids.unsqueeze(1      ).expand(-1					,      self.num_choices					,      -1      ).contiguous()
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=token_type_ids.unsqueeze(1      ).expand(-1					,      self.num_choices					,      -1      ).contiguous()
       SCREAMING_SNAKE_CASE__     :				Tuple									=input_mask.unsqueeze(1      ).expand(-1					,      self.num_choices					,      -1      ).contiguous()
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=model(
           __lowercase					,      attention_mask=__lowercase					,      token_type_ids=__lowercase					,      labels=__lowercase					,      )
       self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.num_choices)      )




 def      __magic_name__	(      self	:							str      )	->						Any:
       SCREAMING_SNAKE_CASE__     :				Tuple									=self.prepare_config_and_inputs()
       (
           (
           SCREAMING_SNAKE_CASE__
       )		,	(
           SCREAMING_SNAKE_CASE__
       )		,	(
           SCREAMING_SNAKE_CASE__
       )		,	(
           SCREAMING_SNAKE_CASE__
       )		,	(
           SCREAMING_SNAKE_CASE__
       )		,	(
           SCREAMING_SNAKE_CASE__
       )		,	(
           SCREAMING_SNAKE_CASE__
       )		,	
       )     :				str									=config_and_inputs
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict






@require_torch
class      __SCREAMING_SNAKE_CASE      (			lowerCamelCase	,				lowerCamelCase	,				unittest.TestCase				):
 snake_case_  							=							(
     (
         MegatronBertModel,
         MegatronBertForMaskedLM,
         MegatronBertForCausalLM,
         MegatronBertForMultipleChoice,
         MegatronBertForNextSentencePrediction,
         MegatronBertForPreTraining,
         MegatronBertForQuestionAnswering,
         MegatronBertForSequenceClassification,
         MegatronBertForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 snake_case_  							=							(
     {
         """feature-extraction""": MegatronBertModel,
         """fill-mask""": MegatronBertForMaskedLM,
         """question-answering""": MegatronBertForQuestionAnswering,
         """text-classification""": MegatronBertForSequenceClassification,
         """text-generation""": MegatronBertForCausalLM,
         """token-classification""": MegatronBertForTokenClassification,
         """zero-shot""": MegatronBertForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 snake_case_  							=							True
 # test_resize_embeddings = False
 snake_case_  							=							False




 def      __magic_name__	(      self	:							List[Any]					,      __lowercase	:							List[str]					,      __lowercase	:							Tuple					,      __lowercase	:							Tuple=False      )	->						Union[str, Any]:
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=super()._prepare_for_class(__lowercase					,      __lowercase					,      return_labels=__lowercase      )

       if return_labels:
             if model_class in get_values(__lowercase      ):
                   SCREAMING_SNAKE_CASE__     :				List[str]									=torch.zeros(
                       (self.model_tester.batch_size, self.model_tester.seq_length)					,      dtype=torch.long					,      device=__lowercase      )
                   SCREAMING_SNAKE_CASE__     :				List[Any]									=torch.zeros(
                       self.model_tester.batch_size					,      dtype=torch.long					,      device=__lowercase      )
       return inputs_dict




 def      __magic_name__	(      self	:							Tuple      )	->						List[Any]:
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=MegatronBertModelTester(self      )
       SCREAMING_SNAKE_CASE__     :				Tuple									=ConfigTester(self					,      config_class=__lowercase					,      hidden_size=37      )




 def      __magic_name__	(      self	:							str      )	->						Dict:
       self.config_tester.run_common_tests()




 def      __magic_name__	(      self	:							Tuple      )	->						List[Any]:
       SCREAMING_SNAKE_CASE__     :				str									=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_model(*__lowercase      )




 def      __magic_name__	(      self	:							List[str]      )	->						Any:
       SCREAMING_SNAKE_CASE__     :				int									=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_masked_lm(*__lowercase      )




 def      __magic_name__	(      self	:							Optional[Any]      )	->						Union[str, Any]:
       SCREAMING_SNAKE_CASE__     :				Dict									=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*__lowercase      )




 def      __magic_name__	(      self	:							Optional[int]      )	->						Dict:
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*__lowercase      )




 def      __magic_name__	(      self	:							Dict      )	->						Optional[int]:
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_pretraining(*__lowercase      )




 def      __magic_name__	(      self	:							int      )	->						Dict:
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_question_answering(*__lowercase      )




 def      __magic_name__	(      self	:							List[Any]      )	->						List[Any]:
       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*__lowercase      )




 def      __magic_name__	(      self	:							Union[str, Any]      )	->						Union[str, Any]:
       SCREAMING_SNAKE_CASE__     :				List[Any]									=self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_token_classification(*__lowercase      )





def  _a(  UpperCamelCase__				:						List[str]							):




      '''simple docstring'''






      return torch.tensor(
          UpperCamelCase__,	dtype=torch.long,	device=UpperCamelCase__,	)


a_					= 1E-4







@require_torch
@require_sentencepiece
@require_tokenizers
class      __SCREAMING_SNAKE_CASE      (			unittest.TestCase				):




 @slow
 @unittest.skip('''Model is not available.'''      )
 def      __magic_name__	(      self	:							Any      )	->						Optional[int]:
       SCREAMING_SNAKE_CASE__     :				Any									='''nvidia/megatron-bert-uncased-345m'''
       if "MYDIR" in os.environ:
             SCREAMING_SNAKE_CASE__     :				Optional[int]									=os.path.join(os.environ['''MYDIR''']					,      __lowercase      )
       SCREAMING_SNAKE_CASE__     :				Any									=MegatronBertModel.from_pretrained(__lowercase      )
       model.to(__lowercase      )
       model.half()
       SCREAMING_SNAKE_CASE__     :				Dict									=_long_tensor([[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]      )
       with torch.no_grad():
             SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=model(__lowercase      )[0]
       SCREAMING_SNAKE_CASE__     :				Dict									=torch.Size((1, 9, 10_24)      )
       self.assertEqual(output.shape					,      __lowercase      )

       SCREAMING_SNAKE_CASE__     :				str									=[-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
       for ii in range(3      ):
             for jj in range(3      ):
                   SCREAMING_SNAKE_CASE__     :				List[Any]									=output[0, ii, jj]
                   SCREAMING_SNAKE_CASE__     :				Tuple									=expected[3 * ii + jj]
                   SCREAMING_SNAKE_CASE__     :				List[str]									='''ii={} jj={} a={} b={}'''.format(__lowercase					,      __lowercase					,      __lowercase					,      __lowercase      )
                   self.assertTrue(math.isclose(__lowercase					,      __lowercase					,      rel_tol=__lowercase					,      abs_tol=__lowercase      )					,      msg=__lowercase      )