
import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    NystromformerForMaskedLM,
							    NystromformerForMultipleChoice,
							    NystromformerForQuestionAnswering,
							    NystromformerForSequenceClassification,
							    NystromformerForTokenClassification,
							    NystromformerModel,
							)
							from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST







class        lowercase	:




							def __init__(       self     ,A__     ,A__=1_3     ,A__=7     ,A__=True     ,A__=True     ,A__=True     ,A__=True     ,A__=9_9     ,A__=3_2     ,A__=5     ,A__=4     ,A__=3_7     ,A__="gelu"     ,A__=0.1     ,A__=0.1     ,A__=5_1_2     ,A__=1_6     ,A__=2     ,A__=0.02     ,A__=3     ,A__=4     ,A__=None     ,):
												lowercase					=		parent
												lowercase					=		batch_size
												lowercase					=		seq_length
												lowercase					=		is_training
												lowercase					=		use_input_mask
												lowercase					=		use_token_type_ids
												lowercase					=		use_labels
												lowercase					=		vocab_size
												lowercase					=		hidden_size
												lowercase					=		num_hidden_layers
												lowercase					=		num_attention_heads
												lowercase					=		intermediate_size
												lowercase					=		hidden_act
												lowercase					=		hidden_dropout_prob
												lowercase					=		attention_probs_dropout_prob
												lowercase					=		max_position_embeddings
												lowercase					=		type_vocab_size
												lowercase					=		type_sequence_label_size
												lowercase					=		initializer_range
												lowercase					=		num_labels
												lowercase					=		num_choices
												lowercase					=		scope




							def 				A__   (       self):
												lowercase					=		ids_tensor([self.batch_size, self.seq_length]     ,self.vocab_size)

												lowercase					=		None
												if self.use_input_mask:
																	lowercase					=		random_attention_mask([self.batch_size, self.seq_length])

												lowercase					=		None
												if self.use_token_type_ids:
																	lowercase					=		ids_tensor([self.batch_size, self.seq_length]     ,self.type_vocab_size)

												lowercase					=		None
												lowercase					=		None
												lowercase					=		None
												if self.use_labels:
																	lowercase					=		ids_tensor([self.batch_size]     ,self.type_sequence_label_size)
																	lowercase					=		ids_tensor([self.batch_size, self.seq_length]     ,self.num_labels)
																	lowercase					=		ids_tensor([self.batch_size]     ,self.num_choices)

												lowercase					=		self.get_config()

												return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




							def 				A__   (       self):
												return NystromformerConfig(
												    vocab_size=self.vocab_size     ,hidden_size=self.hidden_size     ,num_hidden_layers=self.num_hidden_layers     ,num_attention_heads=self.num_attention_heads     ,intermediate_size=self.intermediate_size     ,hidden_act=self.hidden_act     ,hidden_dropout_prob=self.hidden_dropout_prob     ,attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,max_position_embeddings=self.max_position_embeddings     ,type_vocab_size=self.type_vocab_size     ,is_decoder=A__     ,initializer_range=self.initializer_range     ,)




							def 				A__   (       self     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__):
												lowercase					=		NystromformerModel(config=A__)
												model.to(A__)
												model.eval()
												lowercase					=		model(A__     ,attention_mask=A__     ,token_type_ids=A__)
												lowercase					=		model(A__     ,token_type_ids=A__)
												lowercase					=		model(A__)
												self.parent.assertEqual(result.last_hidden_state.shape     ,(self.batch_size, self.seq_length, self.hidden_size))




							def 				A__   (       self     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__):
												lowercase					=		NystromformerForMaskedLM(config=A__)
												model.to(A__)
												model.eval()
												lowercase					=		model(A__     ,attention_mask=A__     ,token_type_ids=A__     ,labels=A__)
												self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.seq_length, self.vocab_size))




							def 				A__   (       self     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__):
												lowercase					=		NystromformerForQuestionAnswering(config=A__)
												model.to(A__)
												model.eval()
												lowercase					=		model(
												    A__     ,attention_mask=A__     ,token_type_ids=A__     ,start_positions=A__     ,end_positions=A__     ,)
												self.parent.assertEqual(result.start_logits.shape     ,(self.batch_size, self.seq_length))
												self.parent.assertEqual(result.end_logits.shape     ,(self.batch_size, self.seq_length))




							def 				A__   (       self     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__):
												lowercase					=		self.num_labels
												lowercase					=		NystromformerForSequenceClassification(A__)
												model.to(A__)
												model.eval()
												lowercase					=		model(A__     ,attention_mask=A__     ,token_type_ids=A__     ,labels=A__)
												self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.num_labels))




							def 				A__   (       self     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__):
												lowercase					=		self.num_labels
												lowercase					=		NystromformerForTokenClassification(config=A__)
												model.to(A__)
												model.eval()
												lowercase					=		model(A__     ,attention_mask=A__     ,token_type_ids=A__     ,labels=A__)
												self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.seq_length, self.num_labels))




							def 				A__   (       self     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__     ,A__):
												lowercase					=		self.num_choices
												lowercase					=		NystromformerForMultipleChoice(config=A__)
												model.to(A__)
												model.eval()
												lowercase					=		input_ids.unsqueeze(1).expand(-1     ,self.num_choices     ,-1).contiguous()
												lowercase					=		token_type_ids.unsqueeze(1).expand(-1     ,self.num_choices     ,-1).contiguous()
												lowercase					=		input_mask.unsqueeze(1).expand(-1     ,self.num_choices     ,-1).contiguous()
												lowercase					=		model(
												    A__     ,attention_mask=A__     ,token_type_ids=A__     ,labels=A__     ,)
												self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.num_choices))





							def 				A__   (       self):
												lowercase					=		self.prepare_config_and_inputs()
												(
												    (
												    lowercase
												)			,		(
												    lowercase
												)			,		(
												    lowercase
												)			,		(
												    lowercase
												)			,		(
												    lowercase
												)			,		(
												    lowercase
												)			,		(
												    lowercase
												)			,		
												)					=		config_and_inputs
												lowercase					=		{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
												return config, inputs_dict







@require_torch
class        lowercase	(      SCREAMING_SNAKE_CASE__ ,						SCREAMING_SNAKE_CASE__ ,						unittest.TestCase	):
							lowercase_ : List[str]       =(
							    (
							        NystromformerModel,
							        NystromformerForMaskedLM,
							        NystromformerForMultipleChoice,
							        NystromformerForQuestionAnswering,
							        NystromformerForSequenceClassification,
							        NystromformerForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowercase_ : Optional[int]       =(
							    {
							        '''feature-extraction''': NystromformerModel,
							        '''fill-mask''': NystromformerForMaskedLM,
							        '''question-answering''': NystromformerForQuestionAnswering,
							        '''text-classification''': NystromformerForSequenceClassification,
							        '''token-classification''': NystromformerForTokenClassification,
							        '''zero-shot''': NystromformerForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							lowercase_ : Any       =False
							lowercase_ : Dict       =False




							def 				A__   (       self):
												lowercase					=		NystromformerModelTester(self)
												lowercase					=		ConfigTester(self     ,config_class=A__     ,hidden_size=3_7)




							def 				A__   (       self):
												self.config_tester.run_common_tests()




							def 				A__   (       self):
												lowercase					=		self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*A__)




							def 				A__   (       self):
												lowercase					=		self.model_tester.prepare_config_and_inputs()
												for type in ["absolute", "relative_key", "relative_key_query"]:
																	lowercase					=		type
																	self.model_tester.create_and_check_model(*A__)




							def 				A__   (       self):
												lowercase					=		self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_masked_lm(*A__)




							def 				A__   (       self):
												lowercase					=		self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_multiple_choice(*A__)




							def 				A__   (       self):
												lowercase					=		self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(*A__)




							def 				A__   (       self):
												lowercase					=		self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(*A__)




							def 				A__   (       self):
												lowercase					=		self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(*A__)





							@slow
							def 				A__   (       self):
												for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	lowercase					=		NystromformerModel.from_pretrained(A__)
																	self.assertIsNotNone(A__)







@require_torch
class        lowercase	(      unittest.TestCase	):




							@slow
							def 				A__   (       self):
												lowercase					=		NystromformerModel.from_pretrained('''uw-madison/nystromformer-512''')
												lowercase					=		torch.tensor([[0, 1, 2, 3, 4, 5]])

												with torch.no_grad():
																	lowercase					=		model(A__)[0]

												lowercase					=		torch.Size((1, 6, 7_6_8))
												self.assertEqual(output.shape     ,A__)

												lowercase					=		torch.tensor(
												    [[[-0.4532, -0.0936, 0.5137], [-0.2676, 0.0628, 0.6186], [-0.3629, -0.1726, 0.4716]]])

												self.assertTrue(torch.allclose(output[:, :3, :3]     ,A__     ,atol=1E-4))





							@slow
							def 				A__   (       self):
												lowercase					=		'''the [MASK] of Belgium is Brussels'''

												lowercase					=		AutoTokenizer.from_pretrained('''uw-madison/nystromformer-512''')
												lowercase					=		NystromformerForMaskedLM.from_pretrained('''uw-madison/nystromformer-512''')

												lowercase					=		tokenizer(A__     ,return_tensors='''pt''')

												with torch.no_grad():
																	lowercase					=		model(encoding.input_ids).logits

												lowercase					=		token_logits[:, 2, :].argmax(-1)[0]

												self.assertEqual(tokenizer.decode(A__)     ,'''capital''')






import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
							import torch_xla.core.xla_model as xm
							import torch_xla.debug.metrics as met







class        lowercase	(      SCREAMING_SNAKE_CASE__	):




							def __init__(       self     ,*A__     ,A__=None     ,A__=None     ,**A__):
												super().__init__(*A__     ,**A__)
												lowercase					=		eval_examples
												lowercase					=		post_process_function




							def 				A__   (       self     ,A__ = None     ,A__=None     ,A__ = None     ,A__ = "eval"     ,**A__     ,):
												lowercase					=		gen_kwargs.copy()
												lowercase					=		(
												    gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length''') is not None else self.args.generation_max_length
												)
												lowercase					=		(
												    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams''') is not None else self.args.generation_num_beams
												)
												lowercase					=		gen_kwargs

												lowercase					=		self.eval_dataset if eval_dataset is None else eval_dataset
												lowercase					=		self.get_eval_dataloader(A__)
												lowercase					=		self.eval_examples if eval_examples is None else eval_examples

												# Temporarily disable metric computation, we will do it in the loop here.
												lowercase					=		self.compute_metrics
												lowercase					=		None
												lowercase					=		time.time()
												lowercase					=		self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	lowercase					=		eval_loop(
																	    A__     ,description='''Evaluation'''     ,prediction_loss_only=True if compute_metrics is None else None     ,ignore_keys=A__     ,metric_key_prefix=A__     ,)
												finally:
																	lowercase					=		compute_metrics
												lowercase					=		self.args.eval_batch_size * self.args.world_size
												if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																	start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
												output.metrics.update(
												    speed_metrics(
												        A__     ,A__     ,num_samples=output.num_samples     ,num_steps=math.ceil(output.num_samples / total_batch_size)     ,))

												if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																	# Only the main node write the results by default
																	lowercase					=		self.post_process_function(A__     ,A__     ,A__)
																	lowercase					=		self.compute_metrics(A__)

																	# Prefix all keys with metric_key_prefix + '_'
																	for key in list(metrics.keys()):
																						if not key.startswith(f'{metric_key_prefix}_'):
																											lowercase					=		metrics.pop(A__)

																	metrics.update(output.metrics)
												else:
																	lowercase					=		output.metrics

												if self.args.should_log:
																	# Only the main node log the results by default
																	self.log(A__)

												if self.args.tpu_metrics_debug or self.args.debug:
																	# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																	xm.master_print(met.metrics_report())

												lowercase					=		self.callback_handler.on_evaluate(self.args     ,self.state     ,self.control     ,A__)
												return metrics





							def 				A__   (       self     ,A__     ,A__     ,A__=None     ,A__ = "test"     ,**A__):
												lowercase					=		gen_kwargs.copy()

												lowercase					=		self.get_test_dataloader(A__)

												# Temporarily disable metric computation, we will do it in the loop here.
												lowercase					=		self.compute_metrics
												lowercase					=		None
												lowercase					=		time.time()
												lowercase					=		self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
												try:
																	lowercase					=		eval_loop(
																	    A__     ,description='''Prediction'''     ,prediction_loss_only=True if compute_metrics is None else None     ,ignore_keys=A__     ,metric_key_prefix=A__     ,)
												finally:
																	lowercase					=		compute_metrics

												lowercase					=		self.args.eval_batch_size * self.args.world_size
												if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																	start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
												output.metrics.update(
												    speed_metrics(
												        A__     ,A__     ,num_samples=output.num_samples     ,num_steps=math.ceil(output.num_samples / total_batch_size)     ,))
												if self.post_process_function is None or self.compute_metrics is None:
																	return output

												lowercase					=		self.post_process_function(A__     ,A__     ,A__     ,'''predict''')
												lowercase					=		self.compute_metrics(A__)

												# Prefix all keys with metric_key_prefix + '_'
												for key in list(metrics.keys()):
																	if not key.startswith(f'{metric_key_prefix}_'):
																						lowercase					=		metrics.pop(A__)
												metrics.update(output.metrics)
												return PredictionOutput(predictions=predictions.predictions     ,label_ids=predictions.label_ids     ,metrics=A__)




