

from __future__ import annotations

import unittest

from transformers import BlenderbotSmallConfig, BlenderbotSmallTokenizer, is_tf_available
from transformers.testing_utils import require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import TFAutoModelForSeqaSeqLM, TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel




@require_tf
class      UpperCamelCase :
    __UpperCamelCase         =BlenderbotSmallConfig
    __UpperCamelCase         ={}
    __UpperCamelCase         ="gelu"

    def __init__(			self :      Union[str, Any]							,    snake_case__ :      List[str]							,    snake_case__ :      List[Any]=1_3							,    snake_case__ :      List[Any]=7							,    snake_case__ :      str=True							,    snake_case__ :      Optional[Any]=False							,    snake_case__ :      Union[str, Any]=9_9							,    snake_case__ :      Optional[int]=3_2							,    snake_case__ :      str=2							,    snake_case__ :      str=4							,    snake_case__ :      Dict=3_7							,    snake_case__ :      Dict=0.1							,    snake_case__ :      Optional[Any]=0.1							,    snake_case__ :      Optional[int]=2_0							,    snake_case__ :      int=2							,    snake_case__ :      str=1							,    snake_case__ :      Union[str, Any]=0							,    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			parent
     SCREAMING_SNAKE_CASE								=			batch_size
     SCREAMING_SNAKE_CASE								=			seq_length
     SCREAMING_SNAKE_CASE								=			is_training
     SCREAMING_SNAKE_CASE								=			use_labels
     SCREAMING_SNAKE_CASE								=			vocab_size
     SCREAMING_SNAKE_CASE								=			hidden_size
     SCREAMING_SNAKE_CASE								=			num_hidden_layers
     SCREAMING_SNAKE_CASE								=			num_attention_heads
     SCREAMING_SNAKE_CASE								=			intermediate_size

     SCREAMING_SNAKE_CASE								=			hidden_dropout_prob
     SCREAMING_SNAKE_CASE								=			attention_probs_dropout_prob
     SCREAMING_SNAKE_CASE								=			max_position_embeddings
     SCREAMING_SNAKE_CASE								=			eos_token_id
     SCREAMING_SNAKE_CASE								=			pad_token_id
     SCREAMING_SNAKE_CASE								=			bos_token_id

    def UpperCamelCase			(			self :      int    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			ids_tensor([self.batch_size, self.seq_length - 1]							,    self.vocab_size    )
     SCREAMING_SNAKE_CASE								=			tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size    )							,    1    )
     SCREAMING_SNAKE_CASE								=			tf.concat([input_ids, eos_tensor]							,    axis=1    )

     SCREAMING_SNAKE_CASE								=			ids_tensor([self.batch_size, self.seq_length]							,    self.vocab_size    )

     SCREAMING_SNAKE_CASE								=			self.config_cls(
         vocab_size=self.vocab_size							,    d_model=self.hidden_size							,    encoder_layers=self.num_hidden_layers							,    decoder_layers=self.num_hidden_layers							,    encoder_attention_heads=self.num_attention_heads							,    decoder_attention_heads=self.num_attention_heads							,    encoder_ffn_dim=self.intermediate_size							,    decoder_ffn_dim=self.intermediate_size							,    dropout=self.hidden_dropout_prob							,    attention_dropout=self.attention_probs_dropout_prob							,    max_position_embeddings=self.max_position_embeddings							,    eos_token_ids=[2]							,    bos_token_id=self.bos_token_id							,    pad_token_id=self.pad_token_id							,    decoder_start_token_id=self.pad_token_id							,    **self.config_updates							,    )
     SCREAMING_SNAKE_CASE								=			prepare_blenderbot_small_inputs_dict(snake_case__							,    snake_case__							,    snake_case__    )
     return config, inputs_dict




    def UpperCamelCase			(			self :      Any							,    snake_case__ :      Tuple							,    snake_case__ :      List[str]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			TFBlenderbotSmallModel(config=snake_case__    ).get_decoder()
     SCREAMING_SNAKE_CASE								=			inputs_dict['input_ids']

     SCREAMING_SNAKE_CASE								=			input_ids[:1, :]
     SCREAMING_SNAKE_CASE								=			inputs_dict['attention_mask'][:1, :]
     SCREAMING_SNAKE_CASE								=			inputs_dict['head_mask']
     SCREAMING_SNAKE_CASE								=			1

     # first forward pass
     SCREAMING_SNAKE_CASE								=			model(snake_case__							,    attention_mask=snake_case__							,    head_mask=snake_case__							,    use_cache=snake_case__    )

     SCREAMING_SNAKE_CASE						,	SCREAMING_SNAKE_CASE								=			outputs.to_tuple()

     # create hypothetical next token and extent to next_input_ids
     SCREAMING_SNAKE_CASE								=			ids_tensor((self.batch_size, 3)							,    config.vocab_size    )
     SCREAMING_SNAKE_CASE								=			tf.cast(ids_tensor((self.batch_size, 3)							,    2    )							,    tf.inta    )

     # append to next input_ids and
     SCREAMING_SNAKE_CASE								=			tf.concat([input_ids, next_tokens]							,    axis=-1    )
     SCREAMING_SNAKE_CASE								=			tf.concat([attention_mask, next_attn_mask]							,    axis=-1    )

     SCREAMING_SNAKE_CASE								=			model(snake_case__							,    attention_mask=snake_case__    )[0]
     SCREAMING_SNAKE_CASE								=			model(snake_case__							,    attention_mask=snake_case__							,    past_key_values=snake_case__    )[0]

     self.parent.assertEqual(next_tokens.shape[1]							,    output_from_past.shape[1]    )

     # select random slice
     SCREAMING_SNAKE_CASE								=			int(ids_tensor((1,)							,    output_from_past.shape[-1]    )    )
     SCREAMING_SNAKE_CASE								=			output_from_no_past[:, -3:, random_slice_idx]
     SCREAMING_SNAKE_CASE								=			output_from_past[:, :, random_slice_idx]

     # test that outputs are equal for slice
     tf.debugging.assert_near(snake_case__							,    snake_case__							,    rtol=1E-3    )


def        __lowerCAmelCase    (       _UpperCamelCase		:  Optional[Any]		,							_UpperCamelCase		:  int		,							_UpperCamelCase		:  List[Any]		,							_UpperCamelCase		:  Optional[int]=None		,							_UpperCamelCase		:  Dict=None		,							_UpperCamelCase		:  Tuple=None		,							_UpperCamelCase		:  Optional[Any]=None		,							_UpperCamelCase		:  List[Any]=None		,							)		->							Union[str, Any]:
 '''simple docstring'''


 if attention_mask is None:
  SCREAMING_SNAKE_CASE								=			tf.cast(tf.math.not_equal(_UpperCamelCase		,							config.pad_token_id	)		,							tf.inta	)
 if decoder_attention_mask is None:
  SCREAMING_SNAKE_CASE								=			tf.concat(
      [
          tf.ones(decoder_input_ids[:, :1].shape		,							dtype=tf.inta	),
          tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]		,							config.pad_token_id	)		,							tf.inta	),
      ]		,							axis=-1		,							)
 if head_mask is None:
  SCREAMING_SNAKE_CASE								=			tf.ones((config.encoder_layers, config.encoder_attention_heads)	)
 if decoder_head_mask is None:
  SCREAMING_SNAKE_CASE								=			tf.ones((config.decoder_layers, config.decoder_attention_heads)	)
 if cross_attn_head_mask is None:
  SCREAMING_SNAKE_CASE								=			tf.ones((config.decoder_layers, config.decoder_attention_heads)	)
 return {
     "input_ids": input_ids,
     "decoder_input_ids": decoder_input_ids,
     "attention_mask": attention_mask,
     "decoder_attention_mask": decoder_attention_mask,
     "head_mask": head_mask,
     "decoder_head_mask": decoder_head_mask,
     "cross_attn_head_mask": cross_attn_head_mask,
 }




@require_tf
class      UpperCamelCase (     SCREAMING_SNAKE_CASE	,			SCREAMING_SNAKE_CASE	,			unittest.TestCase   ):
    __UpperCamelCase         =(
        (TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel) if is_tf_available() else ()
    )
    __UpperCamelCase         =(TFBlenderbotSmallForConditionalGeneration,) if is_tf_available() else ()
    __UpperCamelCase         =(
        {
            "conversational": TFBlenderbotSmallForConditionalGeneration,
            "feature-extraction": TFBlenderbotSmallModel,
            "summarization": TFBlenderbotSmallForConditionalGeneration,
            "text2text-generation": TFBlenderbotSmallForConditionalGeneration,
            "translation": TFBlenderbotSmallForConditionalGeneration,
        }
        if is_tf_available()
        else {}
    )
    __UpperCamelCase         =True
    __UpperCamelCase         =False
    __UpperCamelCase         =False

    def UpperCamelCase			(			self :      List[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			TFBlenderbotSmallModelTester(self    )
     SCREAMING_SNAKE_CASE								=			ConfigTester(self							,    config_class=snake_case__    )

    def UpperCamelCase			(			self :      Any    ):

     """simple docstring"""






     self.config_tester.run_common_tests()




    def UpperCamelCase			(			self :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.model_tester.prepare_config_and_inputs_for_common()
     self.model_tester.check_decoder_model_past_large_inputs(*snake_case__    )




@require_tokenizers
@require_tf
class      UpperCamelCase (     unittest.TestCase   ):
    __UpperCamelCase         =[
        "Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like "
        "  i'm going to throw up.\nand why is that?"
    ]
    __UpperCamelCase         ="facebook/blenderbot_small-90M"

    @cached_property
    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     return BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M'    )

    @cached_property
    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name    )
     return model




    @slow
    def UpperCamelCase			(			self :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.tokenizer(self.src_text							,    return_tensors='tf'    )
     SCREAMING_SNAKE_CASE								=			self.model.generate(
         model_inputs.input_ids							,    attention_mask=model_inputs.attention_mask							,    num_beams=2							,    use_cache=snake_case__							,    )
     SCREAMING_SNAKE_CASE								=			self.tokenizer.batch_decode(generated_ids.numpy()							,    skip_special_tokens=snake_case__    )[0]
     assert generated_words in (
         "i don't know. i just feel like i'm going to throw up. it's not fun.",
         "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
         "i'm not sure. i just feel like i've been in a bad situation.",
     )




import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          NystromformerForMaskedLM,
          NystromformerForMultipleChoice,
          NystromformerForQuestionAnswering,
          NystromformerForSequenceClassification,
          NystromformerForTokenClassification,
          NystromformerModel,
      )
      from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST




class      UpperCamelCase :

    def __init__(			self :      List[str]							,    snake_case__ :      Dict							,    snake_case__ :      Optional[Any]=1_3							,    snake_case__ :      Union[str, Any]=7							,    snake_case__ :      List[str]=True							,    snake_case__ :      Any=True							,    snake_case__ :      List[str]=True							,    snake_case__ :      Optional[Any]=True							,    snake_case__ :      List[str]=9_9							,    snake_case__ :      str=3_2							,    snake_case__ :      Dict=5							,    snake_case__ :      str=4							,    snake_case__ :      int=3_7							,    snake_case__ :      Union[str, Any]="gelu"							,    snake_case__ :      Any=0.1							,    snake_case__ :      Dict=0.1							,    snake_case__ :      Optional[Any]=5_1_2							,    snake_case__ :      List[Any]=1_6							,    snake_case__ :      str=2							,    snake_case__ :      int=0.02							,    snake_case__ :      List[str]=3							,    snake_case__ :      Dict=4							,    snake_case__ :      str=None							,    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			parent
     SCREAMING_SNAKE_CASE								=			batch_size
     SCREAMING_SNAKE_CASE								=			seq_length
     SCREAMING_SNAKE_CASE								=			is_training
     SCREAMING_SNAKE_CASE								=			use_input_mask
     SCREAMING_SNAKE_CASE								=			use_token_type_ids
     SCREAMING_SNAKE_CASE								=			use_labels
     SCREAMING_SNAKE_CASE								=			vocab_size
     SCREAMING_SNAKE_CASE								=			hidden_size
     SCREAMING_SNAKE_CASE								=			num_hidden_layers
     SCREAMING_SNAKE_CASE								=			num_attention_heads
     SCREAMING_SNAKE_CASE								=			intermediate_size
     SCREAMING_SNAKE_CASE								=			hidden_act
     SCREAMING_SNAKE_CASE								=			hidden_dropout_prob
     SCREAMING_SNAKE_CASE								=			attention_probs_dropout_prob
     SCREAMING_SNAKE_CASE								=			max_position_embeddings
     SCREAMING_SNAKE_CASE								=			type_vocab_size
     SCREAMING_SNAKE_CASE								=			type_sequence_label_size
     SCREAMING_SNAKE_CASE								=			initializer_range
     SCREAMING_SNAKE_CASE								=			num_labels
     SCREAMING_SNAKE_CASE								=			num_choices
     SCREAMING_SNAKE_CASE								=			scope

    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			ids_tensor([self.batch_size, self.seq_length]							,    self.vocab_size    )

     SCREAMING_SNAKE_CASE								=			None
     if self.use_input_mask:
      SCREAMING_SNAKE_CASE								=			random_attention_mask([self.batch_size, self.seq_length]    )

     SCREAMING_SNAKE_CASE								=			None
     if self.use_token_type_ids:
      SCREAMING_SNAKE_CASE								=			ids_tensor([self.batch_size, self.seq_length]							,    self.type_vocab_size    )

     SCREAMING_SNAKE_CASE								=			None
     SCREAMING_SNAKE_CASE								=			None
     SCREAMING_SNAKE_CASE								=			None
     if self.use_labels:
      SCREAMING_SNAKE_CASE								=			ids_tensor([self.batch_size]							,    self.type_sequence_label_size    )
      SCREAMING_SNAKE_CASE								=			ids_tensor([self.batch_size, self.seq_length]							,    self.num_labels    )
      SCREAMING_SNAKE_CASE								=			ids_tensor([self.batch_size]							,    self.num_choices    )

     SCREAMING_SNAKE_CASE								=			self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

    def UpperCamelCase			(			self :      Optional[Any]    ):

     """simple docstring"""






     return NystromformerConfig(
         vocab_size=self.vocab_size							,    hidden_size=self.hidden_size							,    num_hidden_layers=self.num_hidden_layers							,    num_attention_heads=self.num_attention_heads							,    intermediate_size=self.intermediate_size							,    hidden_act=self.hidden_act							,    hidden_dropout_prob=self.hidden_dropout_prob							,    attention_probs_dropout_prob=self.attention_probs_dropout_prob							,    max_position_embeddings=self.max_position_embeddings							,    type_vocab_size=self.type_vocab_size							,    is_decoder=snake_case__							,    initializer_range=self.initializer_range							,    )

    def UpperCamelCase			(			self :      List[str]							,    snake_case__ :      Any							,    snake_case__ :      Tuple							,    snake_case__ :      Union[str, Any]							,    snake_case__ :      Dict							,    snake_case__ :      List[str]							,    snake_case__ :      int							,    snake_case__ :      Union[str, Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			NystromformerModel(config=snake_case__    )
     model.to(snake_case__    )
     model.eval()
     SCREAMING_SNAKE_CASE								=			model(snake_case__							,    attention_mask=snake_case__							,    token_type_ids=snake_case__    )
     SCREAMING_SNAKE_CASE								=			model(snake_case__							,    token_type_ids=snake_case__    )
     SCREAMING_SNAKE_CASE								=			model(snake_case__    )
     self.parent.assertEqual(result.last_hidden_state.shape							,    (self.batch_size, self.seq_length, self.hidden_size)    )

    def UpperCamelCase			(			self :      List[Any]							,    snake_case__ :      str							,    snake_case__ :      Dict							,    snake_case__ :      Dict							,    snake_case__ :      Dict							,    snake_case__ :      int							,    snake_case__ :      int							,    snake_case__ :      Tuple    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			NystromformerForMaskedLM(config=snake_case__    )
     model.to(snake_case__    )
     model.eval()
     SCREAMING_SNAKE_CASE								=			model(snake_case__							,    attention_mask=snake_case__							,    token_type_ids=snake_case__							,    labels=snake_case__    )
     self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.seq_length, self.vocab_size)    )

    def UpperCamelCase			(			self :      Union[str, Any]							,    snake_case__ :      Tuple							,    snake_case__ :      Optional[Any]							,    snake_case__ :      int							,    snake_case__ :      Optional[int]							,    snake_case__ :      str							,    snake_case__ :      List[str]							,    snake_case__ :      Optional[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			NystromformerForQuestionAnswering(config=snake_case__    )
     model.to(snake_case__    )
     model.eval()
     SCREAMING_SNAKE_CASE								=			model(
         snake_case__							,    attention_mask=snake_case__							,    token_type_ids=snake_case__							,    start_positions=snake_case__							,    end_positions=snake_case__							,    )
     self.parent.assertEqual(result.start_logits.shape							,    (self.batch_size, self.seq_length)    )
     self.parent.assertEqual(result.end_logits.shape							,    (self.batch_size, self.seq_length)    )

    def UpperCamelCase			(			self :      List[str]							,    snake_case__ :      int							,    snake_case__ :      int							,    snake_case__ :      int							,    snake_case__ :      str							,    snake_case__ :      Optional[int]							,    snake_case__ :      Any							,    snake_case__ :      Union[str, Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.num_labels
     SCREAMING_SNAKE_CASE								=			NystromformerForSequenceClassification(snake_case__    )
     model.to(snake_case__    )
     model.eval()
     SCREAMING_SNAKE_CASE								=			model(snake_case__							,    attention_mask=snake_case__							,    token_type_ids=snake_case__							,    labels=snake_case__    )
     self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.num_labels)    )

    def UpperCamelCase			(			self :      Tuple							,    snake_case__ :      int							,    snake_case__ :      Tuple							,    snake_case__ :      List[str]							,    snake_case__ :      List[Any]							,    snake_case__ :      Optional[Any]							,    snake_case__ :      List[Any]							,    snake_case__ :      Dict    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.num_labels
     SCREAMING_SNAKE_CASE								=			NystromformerForTokenClassification(config=snake_case__    )
     model.to(snake_case__    )
     model.eval()
     SCREAMING_SNAKE_CASE								=			model(snake_case__							,    attention_mask=snake_case__							,    token_type_ids=snake_case__							,    labels=snake_case__    )
     self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.seq_length, self.num_labels)    )

    def UpperCamelCase			(			self :      int							,    snake_case__ :      Optional[Any]							,    snake_case__ :      List[str]							,    snake_case__ :      Union[str, Any]							,    snake_case__ :      Tuple							,    snake_case__ :      Optional[int]							,    snake_case__ :      Any							,    snake_case__ :      List[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.num_choices
     SCREAMING_SNAKE_CASE								=			NystromformerForMultipleChoice(config=snake_case__    )
     model.to(snake_case__    )
     model.eval()
     SCREAMING_SNAKE_CASE								=			input_ids.unsqueeze(1    ).expand(-1							,    self.num_choices							,    -1    ).contiguous()
     SCREAMING_SNAKE_CASE								=			token_type_ids.unsqueeze(1    ).expand(-1							,    self.num_choices							,    -1    ).contiguous()
     SCREAMING_SNAKE_CASE								=			input_mask.unsqueeze(1    ).expand(-1							,    self.num_choices							,    -1    ).contiguous()
     SCREAMING_SNAKE_CASE								=			model(
         snake_case__							,    attention_mask=snake_case__							,    token_type_ids=snake_case__							,    labels=snake_case__							,    )
     self.parent.assertEqual(result.logits.shape							,    (self.batch_size, self.num_choices)    )




    def UpperCamelCase			(			self :      Union[str, Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.prepare_config_and_inputs()
     (
         (
         SCREAMING_SNAKE_CASE
     )						,	(
         SCREAMING_SNAKE_CASE
     )						,	(
         SCREAMING_SNAKE_CASE
     )						,	(
         SCREAMING_SNAKE_CASE
     )						,	(
         SCREAMING_SNAKE_CASE
     )						,	(
         SCREAMING_SNAKE_CASE
     )						,	(
         SCREAMING_SNAKE_CASE
     )						,	
     )								=			config_and_inputs
     SCREAMING_SNAKE_CASE								=			{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
     return config, inputs_dict




@require_torch
class      UpperCamelCase (     SCREAMING_SNAKE_CASE	,			SCREAMING_SNAKE_CASE	,			unittest.TestCase   ):
    __UpperCamelCase         =(
        (
            NystromformerModel,
            NystromformerForMaskedLM,
            NystromformerForMultipleChoice,
            NystromformerForQuestionAnswering,
            NystromformerForSequenceClassification,
            NystromformerForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    __UpperCamelCase         =(
        {
            "feature-extraction": NystromformerModel,
            "fill-mask": NystromformerForMaskedLM,
            "question-answering": NystromformerForQuestionAnswering,
            "text-classification": NystromformerForSequenceClassification,
            "token-classification": NystromformerForTokenClassification,
            "zero-shot": NystromformerForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    __UpperCamelCase         =False
    __UpperCamelCase         =False

    def UpperCamelCase			(			self :      int    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			NystromformerModelTester(self    )
     SCREAMING_SNAKE_CASE								=			ConfigTester(self							,    config_class=snake_case__							,    hidden_size=3_7    )

    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     self.config_tester.run_common_tests()

    def UpperCamelCase			(			self :      List[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*snake_case__    )

    def UpperCamelCase			(			self :      int    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.model_tester.prepare_config_and_inputs()
     for type in ["absolute", "relative_key", "relative_key_query"]:
      SCREAMING_SNAKE_CASE								=			type
      self.model_tester.create_and_check_model(*snake_case__    )

    def UpperCamelCase			(			self :      List[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_masked_lm(*snake_case__    )

    def UpperCamelCase			(			self :      Union[str, Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_multiple_choice(*snake_case__    )

    def UpperCamelCase			(			self :      Tuple    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*snake_case__    )

    def UpperCamelCase			(			self :      Any    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_sequence_classification(*snake_case__    )

    def UpperCamelCase			(			self :      List[Any]    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*snake_case__    )




    @slow
    def UpperCamelCase			(			self :      List[str]    ):

     """simple docstring"""






     for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
      SCREAMING_SNAKE_CASE								=			NystromformerModel.from_pretrained(snake_case__    )
      self.assertIsNotNone(snake_case__    )




@require_torch
class      UpperCamelCase (     unittest.TestCase   ):

    @slow
    def UpperCamelCase			(			self :      int    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			NystromformerModel.from_pretrained('uw-madison/nystromformer-512'    )
     SCREAMING_SNAKE_CASE								=			torch.tensor([[0, 1, 2, 3, 4, 5]]    )

     with torch.no_grad():
      SCREAMING_SNAKE_CASE								=			model(snake_case__    )[0]

     SCREAMING_SNAKE_CASE								=			torch.Size((1, 6, 7_6_8)    )
     self.assertEqual(output.shape							,    snake_case__    )

     SCREAMING_SNAKE_CASE								=			torch.tensor(
         [[[-0.4_532, -0.0_936, 0.5_137], [-0.2_676, 0.0_628, 0.6_186], [-0.3_629, -0.1_726, 0.4_716]]]    )

     self.assertTrue(torch.allclose(output[:, :3, :3]							,    snake_case__							,    atol=1E-4    )    )




    @slow
    def UpperCamelCase			(			self :      Dict    ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE								=			'the [MASK] of Belgium is Brussels'

     SCREAMING_SNAKE_CASE								=			AutoTokenizer.from_pretrained('uw-madison/nystromformer-512'    )
     SCREAMING_SNAKE_CASE								=			NystromformerForMaskedLM.from_pretrained('uw-madison/nystromformer-512'    )

     SCREAMING_SNAKE_CASE								=			tokenizer(snake_case__							,    return_tensors='pt'    )

     with torch.no_grad():
      SCREAMING_SNAKE_CASE								=			model(encoding.input_ids    ).logits

     SCREAMING_SNAKE_CASE								=			token_logits[:, 2, :].argmax(-1    )[0]

     self.assertEqual(tokenizer.decode(snake_case__    )							,    'capital'    )

