

'''simple docstring'''






import math
import os
import re
import sys
import unittest
from pathlib import Path
from typing import Tuple
from unittest.mock import patch

from parameterized import parameterized

from transformers.testing_utils import (
    CaptureStderr,
    ExtendSysPath,
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_torch_dist_unique_port,
    require_apex,
    require_bitsandbytes,
    require_fairscale,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    require_torch_non_multi_gpu,
    slow,
)
from transformers.trainer_callback import TrainerState
from transformers.trainer_utils import set_seed


lowerCAmelCase__      		=			os.path.abspath(os.path.dirname(__file__))
with ExtendSysPath(f'{bindir}/../../examples/pytorch/translation'):
  from run_translation import main  # noqa


set_seed(42)
lowerCAmelCase__      		=			'''sshleifer/student_marian_en_ro_6_1'''
lowerCAmelCase__      		=			'''sshleifer/tiny-mbart'''






@require_torch
class     lowercase_			(lowerCamelCase__   ):




  """simple docstring"""




  def       SCREAMING_SNAKE_CASE   (   self     :  int      ,lowercase__     :  int=False      ,lowercase__     :  Optional[Any]=None      ,lowercase__     :  int=True      ,lowercase__     :  int=True      ,lowercase__     :  Optional[Any]=True      ,lowercase__     :  str=True      ,):
   __lowercase													=				self.run_trainer(
       eval_steps=1      ,max_len=1_2      ,model_name=lowercase__      ,num_train_epochs=1      ,distributed=lowercase__      ,extra_args_str=lowercase__      ,predict_with_generate=lowercase__      ,do_train=lowercase__      ,do_eval=lowercase__      ,do_predict=lowercase__      ,)
   __lowercase													=				TrainerState.load_from_json(os.path.join(lowercase__      ,'''trainer_state.json'''    )    ).log_history

   if not do_eval:
    return

   __lowercase													=				[log for log in logs if '''eval_loss''' in log.keys()]

   __lowercase													=				eval_metrics[0]
   if predict_with_generate:
    assert "eval_bleu" in first_step_stats

    __lowercase													=				eval_metrics[-1]
    assert isinstance(last_step_stats['''eval_bleu''']      ,lowercase__    )
    assert not math.isnan(float(last_step_stats['''eval_loss''']    )    ), "eval_loss must not be `nan`"






  @require_torch_non_multi_gpu
  def       SCREAMING_SNAKE_CASE   (   self     :  str    ):
   self.run_seqaseq_quick()






  @require_torch_multi_gpu
  def       SCREAMING_SNAKE_CASE   (   self     :  Optional[Any]    ):
   self.run_seqaseq_quick(distributed=lowercase__    )






  @require_torch_multi_gpu
  def       SCREAMING_SNAKE_CASE   (   self     :  Dict    ):
   self.run_seqaseq_quick(distributed=lowercase__    )






  @unittest.skip('''Requires an update of the env running those tests'''    )
  @require_torch_multi_gpu
  @require_fairscale
  def       SCREAMING_SNAKE_CASE   (   self     :  Optional[Any]    ):
   self.run_seqaseq_quick(distributed=lowercase__      ,extra_args_str='''--sharded_ddp simple'''    )






  @unittest.skip('''Requires an update of the env running those tests'''    )
  @require_torch_multi_gpu
  @require_fairscale
  def       SCREAMING_SNAKE_CASE   (   self     :  int    ):
   self.run_seqaseq_quick(distributed=lowercase__      ,extra_args_str='''--sharded_ddp simple --fp16'''    )






  @unittest.skip('''Requires an update of the env running those tests'''    )
  @require_torch_multi_gpu
  @require_fairscale
  def       SCREAMING_SNAKE_CASE   (   self     :  Dict    ):
   self.run_seqaseq_quick(distributed=lowercase__      ,extra_args_str='''--sharded_ddp zero_dp_2'''      ,predict_with_generate=lowercase__    )






  @unittest.skip('''Requires an update of the env running those tests'''    )
  @require_torch_multi_gpu
  @require_fairscale
  def       SCREAMING_SNAKE_CASE   (   self     :  Optional[int]    ):
   self.run_seqaseq_quick(
       distributed=lowercase__      ,extra_args_str='''--sharded_ddp zero_dp_2 --fp16'''      ,predict_with_generate=lowercase__    )






  @require_apex
  @require_torch_gpu
  def       SCREAMING_SNAKE_CASE   (   self     :  Tuple    ):
   # XXX: apex breaks the trainer if it's run twice e.g. run_seq2seq.main() from the same
   # program and it breaks other tests that run from the same pytest worker, therefore until this is
   # sorted out it must be run only in an external program, that is distributed=True in this
   # test and only under one or more gpus - if we want cpu will need to make a special test
   #
   # specifically to the problem traced it to self.optimizer.step() - if it's run 2nd time via
   # 2nd main() call it botches the future eval.
   #
   self.run_seqaseq_quick(distributed=lowercase__      ,extra_args_str='''--fp16 --fp16_backend=apex'''    )
   # test 2nd time - was getting eval_loss': nan'
   # to reproduce the problem set distributed=False
   self.run_seqaseq_quick(distributed=lowercase__      ,extra_args_str='''--fp16 --fp16_backend=apex'''    )






  @parameterized.expand(['''base''', '''low''', '''high''', '''mixed''']    )
  @require_torch_multi_gpu
  def       SCREAMING_SNAKE_CASE   (   self     :  Dict      ,lowercase__     :  Dict    ):
   # as each sub-test is slow-ish split into multiple sub-tests to avoid CI timeout
   __lowercase													=				{
       # test with the default log_level - should be info and thus log info once
       '''base''': {'''extra_args_str''': '''''', '''n_matches''': 1},
       # test with low log_level and log_level_replica - should be noisy on all processes
       # now the info string should appear twice on 2 processes
       '''low''': {'''extra_args_str''': '''--log_level debug --log_level_replica debug''', '''n_matches''': 2},
       # test with high log_level and low log_level_replica
       # now the info string should appear once only on the replica
       '''high''': {'''extra_args_str''': '''--log_level error --log_level_replica debug''', '''n_matches''': 1},
       # test with high log_level and log_level_replica - should be quiet on all processes
       '''mixed''': {'''extra_args_str''': '''--log_level error --log_level_replica error''', '''n_matches''': 0},
   }

   __lowercase													=				experiments[experiment_id]
   __lowercase													=				{'''distributed''': True, '''predict_with_generate''': False, '''do_eval''': False, '''do_predict''': False}
   __lowercase													=				'''Running training'''
   with CaptureStderr() as cl:
    self.run_seqaseq_quick(**lowercase__      ,extra_args_str=data['''extra_args_str''']    )
   __lowercase													=				len(re.findall(lowercase__      ,cl.err    )    )
   self.assertEqual(lowercase__      ,data['''n_matches''']    )






  @slow
  def       SCREAMING_SNAKE_CASE   (   self     :  Tuple    ):
   __lowercase													=				self.run_trainer(
       eval_steps=2      ,max_len=1_2_8      ,model_name=lowercase__      ,learning_rate=3e-4      ,num_train_epochs=1_0      ,distributed=lowercase__      ,)

   # Check metrics
   __lowercase													=				TrainerState.load_from_json(os.path.join(lowercase__      ,'''trainer_state.json'''    )    ).log_history
   __lowercase													=				[log for log in logs if '''eval_loss''' in log.keys()]
   __lowercase													=				eval_metrics[0]
   __lowercase													=				eval_metrics[-1]

   assert first_step_stats["eval_loss"] > last_step_stats["eval_loss"], "model learned nothing"
   assert isinstance(last_step_stats['''eval_bleu''']      ,lowercase__    )

   # test if do_predict saves generations and metrics
   __lowercase													=				os.listdir(lowercase__    )
   __lowercase													=				{os.path.basename(lowercase__    ) for p in contents}
   assert "generated_predictions.txt" in contents
   assert "predict_results.json" in contents






  @slow
  @require_bitsandbytes
  def       SCREAMING_SNAKE_CASE   (   self     :  Optional[Any]    ):
   from transformers.training_args import OptimizerNames

   def train_and_return_metrics(lowercase__     :  str    ) -> Tuple[int, float]:
    __lowercase													=				'''--skip_memory_metrics 0'''

    __lowercase													=				self.run_trainer(
        max_len=1_2_8      ,model_name=lowercase__      ,learning_rate=3e-4      ,num_train_epochs=1      ,optim=lowercase__      ,distributed=lowercase__      ,extra_args_str=lowercase__      ,do_eval=lowercase__      ,do_predict=lowercase__      ,n_gpus_to_use=1      ,)

    # Check metrics
    __lowercase													=				TrainerState.load_from_json(Path(lowercase__      ,'''trainer_state.json'''    )    ).log_history
    __lowercase													=				int(logs[0]['''train_mem_gpu_peaked_delta'''] / 2**2_0    )
    __lowercase													=				int(logs[0]['''train_mem_gpu_alloc_delta'''] / 2**2_0    )

    __lowercase													=				logs[0]['''train_loss''']
    return gpu_peak_mem_mb, gpu_alloc_mem_mb, loss

   __lowercase						,			__lowercase						,			__lowercase													=				train_and_return_metrics(OptimizerNames.ADAMW_TORCH.value    )
   __lowercase						,			__lowercase						,			__lowercase													=				train_and_return_metrics(OptimizerNames.ADAMW_BNB.value    )

   __lowercase													=				gpu_alloc_mem_orig - gpu_alloc_mem_bnb

   __lowercase													=				gpu_peak_mem_orig + gpu_alloc_mem_orig
   __lowercase													=				gpu_peak_mem_bnb + gpu_alloc_mem_bnb
   __lowercase													=				gpu_total_mem_orig - gpu_total_mem_bnb

   # sshleifer/student_marian_en_ro_6_1 has 54M parameter, 29M of which is `nn.Embedding` which
   # doesn't get quantized and remains in fp32. Therefore we only have 25M parameters quantized
   # in 2 bytes and the diff in optim memory usage is derived as so:
   #
   # - normal 25*8=~200MB (8 bytes per param)
   # - bnb    25*2= ~50MB (2 bytes per param)
   #
   # Thus we should expect ~150MB total memory saved.
   #
   # Peak memory should be the same - the total should be different by about that same margin
   #
   # After leaving a small margin to accommodate for differences between gpus let's check
   # that we have at least 120MB in savings
   __lowercase													=				1_2_0

   # uncomment the following if this test starts failing - requires py38 for a new print feature
   # gpu_peak_mem_diff = gpu_peak_mem_orig - gpu_peak_mem_bnb
   # print(f"{gpu_alloc_mem_orig=}MB {gpu_peak_mem_orig=}MB {gpu_alloc_mem_orig+gpu_peak_mem_orig=}MB")
   # print(f" {gpu_alloc_mem_bnb=}MB  {gpu_peak_mem_bnb=}MB  {gpu_alloc_mem_bnb+gpu_peak_mem_bnb=}MB")
   # print(f"{gpu_alloc_mem_diff=}MB")
   # print(f"{gpu_peak_mem_diff=}MB")
   # print(f"{gpu_total_mem_orig=}MB, {gpu_total_mem_bnb=}MB")
   # print(f"{gpu_total_mem_diff=}MB, {gpu_total_mem_diff=}MB")

   self.assertGreater(
       lowercase__      ,lowercase__      ,'''should use ~150MB less alloc gpu memory with BNB, compared to without it for this model but got'''
       F" a difference of {gpu_alloc_mem_diff}MB, with gpu_alloc_mem_orig={gpu_alloc_mem_orig}MB and"
       F" gpu_alloc_mem_bnb={gpu_alloc_mem_bnb}MB"      ,)

   self.assertGreater(
       lowercase__      ,lowercase__      ,'''should use ~150MB less total gpu memory with BNB, compared to without it for this model but got'''
       F" a difference of {gpu_total_mem_diff}MB, with gpu_total_mem_orig={gpu_total_mem_orig}MB and"
       F" gpu_total_mem_bnb={gpu_total_mem_bnb}MB"      ,)

   self.assertEqual(
       lowercase__      ,lowercase__      ,F"loss should be the same, but got loss_orig={loss_orig}, loss_bnb={loss_bnb}"    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Any      ,lowercase__     :  int      ,lowercase__     :  str      ,lowercase__     :  int      ,lowercase__     :  float = 3e-3      ,lowercase__     :  str = "adafactor"      ,lowercase__     :  bool = False      ,lowercase__     :  str = None      ,lowercase__     :  int = 0      ,lowercase__     :  bool = True      ,lowercase__     :  bool = True      ,lowercase__     :  bool = True      ,lowercase__     :  bool = True      ,lowercase__     :  int = None      ,):
   __lowercase													=				self.test_file_dir / '''../fixtures/tests_samples/wmt_en_ro'''
   __lowercase													=				self.get_auto_remove_tmp_dir()
   __lowercase													=				F"\n            --model_name_or_path {model_name}\n            --train_file {data_dir}/train.json\n            --validation_file {data_dir}/val.json\n            --test_file {data_dir}/test.json\n            --output_dir {output_dir}\n            --overwrite_output_dir\n            --max_train_samples 8\n            --max_source_length {max_len}\n            --max_target_length {max_len}\n            --do_train\n            --num_train_epochs {str(lowercase__    )}\n            --per_device_train_batch_size 4\n            --learning_rate {learning_rate}\n            --warmup_steps 8\n            --logging_steps 0\n            --logging_strategy no\n            --save_steps {str(lowercase__    )}\n            --group_by_length\n            --label_smoothing_factor 0.1\n            --target_lang ro_RO\n            --source_lang en_XX\n        ".split()

   __lowercase													=				F"\n            --do_eval\n            --per_device_eval_batch_size 4\n            --max_eval_samples 8\n            --val_max_target_length {max_len}\n            --evaluation_strategy steps\n            --eval_steps {str(lowercase__    )}\n        ".split()

   __lowercase													=				'''
            --do_predict
        '''.split()

   __lowercase													=				[]
   if do_train:
    args += args_train

   if do_eval:
    args += args_eval

   if do_predict:
    args += args_predict

   if predict_with_generate:
    args += "--predict_with_generate".split()

   if do_train:
    if optim == "adafactor":
     args += "--adafactor".split()
    else:
     args += F"--optim {optim}".split()

   if extra_args_str is not None:
    args += extra_args_str.split()

   if distributed:
    if n_gpus_to_use is None:
     __lowercase													=				get_gpu_count()
    __lowercase													=				get_torch_dist_unique_port()
    __lowercase													=				F"\n                -m torch.distributed.run\n                --nproc_per_node={n_gpus_to_use}\n                --master_port={master_port}\n                {self.examples_dir_str}/pytorch/translation/run_translation.py\n            ".split()
    __lowercase													=				[sys.executable] + distributed_args + args
    # keep for quick debug
    # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
    execute_subprocess_async(lowercase__      ,env=self.get_env()    )
   else:
    __lowercase													=				['''run_translation.py'''] + args
    with patch.object(lowercase__      ,'''argv'''      ,lowercase__    ):
     main()

   return output_dir




'''simple docstring'''






from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import (
     TFFunnelBaseModel,
     TFFunnelForMaskedLM,
     TFFunnelForMultipleChoice,
     TFFunnelForPreTraining,
     TFFunnelForQuestionAnswering,
     TFFunnelForSequenceClassification,
     TFFunnelForTokenClassification,
     TFFunnelModel,
 )






class     lowercase_			:




  """simple docstring"""




  def __init__(   self     :  Union[str, Any]      ,lowercase__     :  Union[str, Any]      ,lowercase__     :  List[Any]=1_3      ,lowercase__     :  Union[str, Any]=7      ,lowercase__     :  List[Any]=True      ,lowercase__     :  str=True      ,lowercase__     :  Any=True      ,lowercase__     :  Union[str, Any]=True      ,lowercase__     :  Tuple=9_9      ,lowercase__     :  List[str]=[1, 1, 2]      ,lowercase__     :  List[Any]=1      ,lowercase__     :  List[Any]=3_2      ,lowercase__     :  int=4      ,lowercase__     :  Tuple=8      ,lowercase__     :  Tuple=3_7      ,lowercase__     :  str="gelu_new"      ,lowercase__     :  Dict=0.1      ,lowercase__     :  Optional[int]=0.1      ,lowercase__     :  Optional[Any]=0.0      ,lowercase__     :  Tuple=5_1_2      ,lowercase__     :  Any=3      ,lowercase__     :  List[str]=0.0_2      ,lowercase__     :  List[Any]=3      ,lowercase__     :  List[Any]=4      ,lowercase__     :  Optional[Any]=None      ,lowercase__     :  Tuple=False      ,):
   __lowercase													=				parent
   __lowercase													=				batch_size
   __lowercase													=				seq_length
   __lowercase													=				is_training
   __lowercase													=				use_input_mask
   __lowercase													=				use_token_type_ids
   __lowercase													=				use_labels
   __lowercase													=				vocab_size
   __lowercase													=				block_sizes
   __lowercase													=				num_decoder_layers
   __lowercase													=				d_model
   __lowercase													=				n_head
   __lowercase													=				d_head
   __lowercase													=				d_inner
   __lowercase													=				hidden_act
   __lowercase													=				hidden_dropout
   __lowercase													=				attention_dropout
   __lowercase													=				activation_dropout
   __lowercase													=				max_position_embeddings
   __lowercase													=				type_vocab_size
   __lowercase													=				2
   __lowercase													=				num_labels
   __lowercase													=				num_choices
   __lowercase													=				scope
   __lowercase													=				initializer_std

   # Used in the tests to check the size of the first attention layer
   __lowercase													=				n_head
   # Used in the tests to check the size of the first hidden state
   __lowercase													=				self.d_model
   # Used in the tests to check the number of output hidden states/attentions
   __lowercase													=				sum(self.block_sizes    ) + (0 if base else self.num_decoder_layers)
   # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
   # the last hidden state of the first block (which is the first hidden state of the decoder).
   if not base:
    __lowercase													=				self.num_hidden_layers + 2






  def       SCREAMING_SNAKE_CASE   (   self     :  List[Any]    ):
   __lowercase													=				ids_tensor([self.batch_size, self.seq_length]      ,self.vocab_size    )

   __lowercase													=				None
   if self.use_input_mask:
    __lowercase													=				random_attention_mask([self.batch_size, self.seq_length]    )

   __lowercase													=				None
   if self.use_token_type_ids:
    __lowercase													=				ids_tensor([self.batch_size, self.seq_length]      ,self.type_vocab_size    )

   __lowercase													=				None
   __lowercase													=				None
   __lowercase													=				None
   if self.use_labels:
    __lowercase													=				ids_tensor([self.batch_size]      ,self.type_sequence_label_size    )
    __lowercase													=				ids_tensor([self.batch_size, self.seq_length]      ,self.num_labels    )
    __lowercase													=				ids_tensor([self.batch_size]      ,self.num_choices    )

   __lowercase													=				FunnelConfig(
       vocab_size=self.vocab_size      ,block_sizes=self.block_sizes      ,num_decoder_layers=self.num_decoder_layers      ,d_model=self.d_model      ,n_head=self.n_head      ,d_head=self.d_head      ,d_inner=self.d_inner      ,hidden_act=self.hidden_act      ,hidden_dropout=self.hidden_dropout      ,attention_dropout=self.attention_dropout      ,activation_dropout=self.activation_dropout      ,max_position_embeddings=self.max_position_embeddings      ,type_vocab_size=self.type_vocab_size      ,initializer_std=self.initializer_std      ,)

   return (
       config,
       input_ids,
       token_type_ids,
       input_mask,
       sequence_labels,
       token_labels,
       choice_labels,
   )






  def       SCREAMING_SNAKE_CASE   (   self     :  int      ,lowercase__     :  List[str]      ,lowercase__     :  Union[str, Any]      ,lowercase__     :  int      ,lowercase__     :  List[str]      ,lowercase__     :  Optional[int]      ,lowercase__     :  List[Any]      ,lowercase__     :  List[str]      ,):
   __lowercase													=				TFFunnelModel(config=lowercase__    )
   __lowercase													=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
   __lowercase													=				model(lowercase__    )

   __lowercase													=				[input_ids, input_mask]
   __lowercase													=				model(lowercase__    )

   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.last_hidden_state.shape      ,(self.batch_size, self.seq_length, self.d_model)    )

   __lowercase													=				False
   __lowercase													=				TFFunnelModel(config=lowercase__    )
   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.last_hidden_state.shape      ,(self.batch_size, self.seq_length, self.d_model)    )

   __lowercase													=				False
   __lowercase													=				TFFunnelModel(config=lowercase__    )
   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.last_hidden_state.shape      ,(self.batch_size, self.seq_length, self.d_model)    )






  def       SCREAMING_SNAKE_CASE   (   self     :  int      ,lowercase__     :  List[Any]      ,lowercase__     :  Tuple      ,lowercase__     :  Dict      ,lowercase__     :  Any      ,lowercase__     :  Optional[Any]      ,lowercase__     :  Union[str, Any]      ,lowercase__     :  Dict      ,):
   __lowercase													=				TFFunnelBaseModel(config=lowercase__    )
   __lowercase													=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
   __lowercase													=				model(lowercase__    )

   __lowercase													=				[input_ids, input_mask]
   __lowercase													=				model(lowercase__    )

   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.last_hidden_state.shape      ,(self.batch_size, 2, self.d_model)    )

   __lowercase													=				False
   __lowercase													=				TFFunnelBaseModel(config=lowercase__    )
   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.last_hidden_state.shape      ,(self.batch_size, 3, self.d_model)    )

   __lowercase													=				False
   __lowercase													=				TFFunnelBaseModel(config=lowercase__    )
   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.last_hidden_state.shape      ,(self.batch_size, 2, self.d_model)    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Tuple      ,lowercase__     :  Any      ,lowercase__     :  Any      ,lowercase__     :  List[Any]      ,lowercase__     :  Optional[int]      ,lowercase__     :  Dict      ,lowercase__     :  int      ,lowercase__     :  List[str]      ,):
   __lowercase													=				TFFunnelForPreTraining(config=lowercase__    )
   __lowercase													=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.seq_length)    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Union[str, Any]      ,lowercase__     :  Union[str, Any]      ,lowercase__     :  Optional[Any]      ,lowercase__     :  Dict      ,lowercase__     :  List[Any]      ,lowercase__     :  Tuple      ,lowercase__     :  Optional[Any]      ,lowercase__     :  Tuple      ,):
   __lowercase													=				TFFunnelForMaskedLM(config=lowercase__    )
   __lowercase													=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.seq_length, self.vocab_size)    )






  def       SCREAMING_SNAKE_CASE   (   self     :  List[Any]      ,lowercase__     :  Tuple      ,lowercase__     :  Optional[Any]      ,lowercase__     :  List[Any]      ,lowercase__     :  List[Any]      ,lowercase__     :  str      ,lowercase__     :  Union[str, Any]      ,lowercase__     :  Tuple      ,):
   __lowercase													=				self.num_labels
   __lowercase													=				TFFunnelForSequenceClassification(config=lowercase__    )
   __lowercase													=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.num_labels)    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Dict      ,lowercase__     :  List[str]      ,lowercase__     :  Optional[Any]      ,lowercase__     :  List[str]      ,lowercase__     :  List[str]      ,lowercase__     :  List[str]      ,lowercase__     :  str      ,lowercase__     :  Tuple      ,):
   __lowercase													=				self.num_choices
   __lowercase													=				TFFunnelForMultipleChoice(config=lowercase__    )
   __lowercase													=				tf.tile(tf.expand_dims(lowercase__      ,1    )      ,(1, self.num_choices, 1)    )
   __lowercase													=				tf.tile(tf.expand_dims(lowercase__      ,1    )      ,(1, self.num_choices, 1)    )
   __lowercase													=				tf.tile(tf.expand_dims(lowercase__      ,1    )      ,(1, self.num_choices, 1)    )
   __lowercase													=				{
       '''input_ids''': multiple_choice_inputs_ids,
       '''attention_mask''': multiple_choice_input_mask,
       '''token_type_ids''': multiple_choice_token_type_ids,
   }
   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.num_choices)    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Optional[int]      ,lowercase__     :  Optional[Any]      ,lowercase__     :  List[Any]      ,lowercase__     :  List[str]      ,lowercase__     :  List[Any]      ,lowercase__     :  Dict      ,lowercase__     :  Any      ,lowercase__     :  Union[str, Any]      ,):
   __lowercase													=				self.num_labels
   __lowercase													=				TFFunnelForTokenClassification(config=lowercase__    )
   __lowercase													=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.seq_length, self.num_labels)    )






  def       SCREAMING_SNAKE_CASE   (   self     :  List[Any]      ,lowercase__     :  Optional[Any]      ,lowercase__     :  Optional[int]      ,lowercase__     :  Dict      ,lowercase__     :  List[str]      ,lowercase__     :  str      ,lowercase__     :  Tuple      ,lowercase__     :  Any      ,):
   __lowercase													=				TFFunnelForQuestionAnswering(config=lowercase__    )
   __lowercase													=				{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
   __lowercase													=				model(lowercase__    )
   self.parent.assertEqual(result.start_logits.shape      ,(self.batch_size, self.seq_length)    )
   self.parent.assertEqual(result.end_logits.shape      ,(self.batch_size, self.seq_length)    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Any    ):
   __lowercase													=				self.prepare_config_and_inputs()
   (
       (
       __lowercase
   )						,			(
       __lowercase
   )						,			(
       __lowercase
   )						,			(
       __lowercase
   )						,			(
       __lowercase
   )						,			(
       __lowercase
   )						,			(
       __lowercase
   )						,			
   )													=				config_and_inputs
   __lowercase													=				{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
   return config, inputs_dict






@require_tf
class     lowercase_			(lowerCamelCase__       , lowerCamelCase__       , unittest.TestCase   ):




  """simple docstring"""




  SCREAMING_SNAKE_CASE					:						int           =    (
      (
          TFFunnelModel,
          TFFunnelForMaskedLM,
          TFFunnelForPreTraining,
          TFFunnelForQuestionAnswering,
          TFFunnelForTokenClassification,
      )
      if is_tf_available()
      else ()
  )
  SCREAMING_SNAKE_CASE					:						Union[str, Any]           =    (
      {
          'feature-extraction': (TFFunnelBaseModel, TFFunnelModel),
          'fill-mask': TFFunnelForMaskedLM,
          'question-answering': TFFunnelForQuestionAnswering,
          'text-classification': TFFunnelForSequenceClassification,
          'token-classification': TFFunnelForTokenClassification,
          'zero-shot': TFFunnelForSequenceClassification,
      }
      if is_tf_available()
      else {}
  )
  SCREAMING_SNAKE_CASE					:						Tuple           =    False
  SCREAMING_SNAKE_CASE					:						Any           =    False






  def       SCREAMING_SNAKE_CASE   (   self     :  str    ):
   __lowercase													=				TFFunnelModelTester(self    )
   __lowercase													=				ConfigTester(self      ,config_class=lowercase__    )






  def       SCREAMING_SNAKE_CASE   (   self     :  str    ):
   self.config_tester.run_common_tests()






  def       SCREAMING_SNAKE_CASE   (   self     :  Optional[int]    ):
   __lowercase													=				self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_model(*lowercase__    )






  def       SCREAMING_SNAKE_CASE   (   self     :  List[Any]    ):
   __lowercase													=				self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_pretraining(*lowercase__    )






  def       SCREAMING_SNAKE_CASE   (   self     :  List[str]    ):
   __lowercase													=				self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_masked_lm(*lowercase__    )






  def       SCREAMING_SNAKE_CASE   (   self     :  List[Any]    ):
   __lowercase													=				self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_token_classification(*lowercase__    )






  def       SCREAMING_SNAKE_CASE   (   self     :  int    ):
   __lowercase													=				self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_question_answering(*lowercase__    )






@require_tf
class     lowercase_			(lowerCamelCase__       , unittest.TestCase   ):




  """simple docstring"""




  SCREAMING_SNAKE_CASE					:						List[Any]           =    (
      (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
  )
  SCREAMING_SNAKE_CASE					:						Dict           =    False
  SCREAMING_SNAKE_CASE					:						List[str]           =    False






  def       SCREAMING_SNAKE_CASE   (   self     :  str    ):
   __lowercase													=				TFFunnelModelTester(self      ,base=lowercase__    )
   __lowercase													=				ConfigTester(self      ,config_class=lowercase__    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Any    ):
   self.config_tester.run_common_tests()






  def       SCREAMING_SNAKE_CASE   (   self     :  List[str]    ):
   __lowercase													=				self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_base_model(*lowercase__    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Dict    ):
   __lowercase													=				self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_sequence_classification(*lowercase__    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Any    ):
   __lowercase													=				self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_multiple_choice(*lowercase__    )

