from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import (
        TFRoFormerForCausalLM,
        TFRoFormerForMaskedLM,
        TFRoFormerForMultipleChoice,
        TFRoFormerForQuestionAnswering,
        TFRoFormerForSequenceClassification,
        TFRoFormerForTokenClassification,
        TFRoFormerModel,
    )
    from transformers.models.roformer.modeling_tf_roformer import (
        TFRoFormerSelfAttention,
        TFRoFormerSinusoidalPositionalEmbedding,
    )



class  SCREAMING_SNAKE_CASE_ :
  '''simple docstring'''


  def __init__(				self    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					Dict=13      ,	SCREAMING_SNAKE_CASE__    :					Any=7      ,	SCREAMING_SNAKE_CASE__    :					Dict=True      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]=True      ,	SCREAMING_SNAKE_CASE__    :					List[Any]=True      ,	SCREAMING_SNAKE_CASE__    :					str=True      ,	SCREAMING_SNAKE_CASE__    :					int=99      ,	SCREAMING_SNAKE_CASE__    :					List[Any]=32      ,	SCREAMING_SNAKE_CASE__    :					List[Any]=2      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]=4      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]=37      ,	SCREAMING_SNAKE_CASE__    :					List[Any]="gelu"      ,	SCREAMING_SNAKE_CASE__    :					List[str]=0.1      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]=0.1      ,	SCREAMING_SNAKE_CASE__    :					List[str]=5_12      ,	SCREAMING_SNAKE_CASE__    :					Any=16      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]=2      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]=0.0_2      ,	SCREAMING_SNAKE_CASE__    :					int=3      ,	SCREAMING_SNAKE_CASE__    :					str=4      ,	SCREAMING_SNAKE_CASE__    :					List[str]=None      ,	)    ->				List[Any]:
         A						:     List[str]         =parent
         A						:     Optional[Any]         =13
         A						:     List[str]         =7
         A						:     Optional[int]         =True
         A						:     Union[str, Any]         =True
         A						:     Optional[int]         =True
         A						:     Tuple         =True
         A						:     List[str]         =99
         A						:     Optional[int]         =32
         A						:     Any         =2
         A						:     str         =4
         A						:     List[Any]         =37
         A						:     Union[str, Any]         ='gelu'
         A						:     Tuple         =0.1
         A						:     Optional[Any]         =0.1
         A						:     Any         =5_12
         A						:     Tuple         =16
         A						:     Any         =2
         A						:     List[Any]         =0.0_2
         A						:     str         =3
         A						:     Optional[int]         =4
         A						:     Dict         =None







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Any     )    ->				List[str]:
         A						:     List[Any]         =ids_tensor([self.batch_size, self.seq_length]      ,	self.vocab_size     )

         A						:     Optional[int]         =None
         if self.use_input_mask:
                A						:     Union[str, Any]         =random_attention_mask([self.batch_size, self.seq_length]     )

         A						:     Optional[int]         =None
         if self.use_token_type_ids:
                A						:     Optional[Any]         =ids_tensor([self.batch_size, self.seq_length]      ,	self.type_vocab_size     )

         A						:     str         =None
         A						:     Any         =None
         A						:     List[Any]         =None
         if self.use_labels:
                A						:     int         =ids_tensor([self.batch_size]      ,	self.type_sequence_label_size     )
                A						:     List[Any]         =ids_tensor([self.batch_size, self.seq_length]      ,	self.num_labels     )
                A						:     Optional[int]         =ids_tensor([self.batch_size]      ,	self.num_choices     )

         A						:     Optional[int]         =RoFormerConfig(
             vocab_size=self.vocab_size      ,	hidden_size=self.hidden_size      ,	num_hidden_layers=self.num_hidden_layers      ,	num_attention_heads=self.num_attention_heads      ,	intermediate_size=self.intermediate_size      ,	hidden_act=self.hidden_act      ,	hidden_dropout_prob=self.hidden_dropout_prob      ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,	max_position_embeddings=self.max_position_embeddings      ,	type_vocab_size=self.type_vocab_size      ,	initializer_range=self.initializer_range      ,	return_dict=SCREAMING_SNAKE_CASE__      ,	)

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]      ,	SCREAMING_SNAKE_CASE__    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					List[str]      ,	SCREAMING_SNAKE_CASE__    :					Dict     )    ->				List[str]:
         A						:     str         =TFRoFormerModel(config=SCREAMING_SNAKE_CASE__     )
         A						:     List[Any]         ={'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

         A						:     int         =[input_ids, input_mask]
         A						:     Tuple         =model(SCREAMING_SNAKE_CASE__     )

         A						:     Dict         =model(SCREAMING_SNAKE_CASE__     )

         self.parent.assertEqual(result.last_hidden_state.shape      ,	(self.batch_size, self.seq_length, self.hidden_size)     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Any      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					List[str]     )    ->				Dict:
         A						:     int         =True
         A						:     int         =TFRoFormerForCausalLM(config=SCREAMING_SNAKE_CASE__     )
         A						:     Optional[Any]         ={
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }
         A						:     str         =model(SCREAMING_SNAKE_CASE__     )['logits']
         self.parent.assertListEqual(
             list(prediction_scores.numpy().shape     )      ,	[self.batch_size, self.seq_length, self.vocab_size]     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]     )    ->				Union[str, Any]:
         A						:     Any         =TFRoFormerForMaskedLM(config=SCREAMING_SNAKE_CASE__     )
         A						:     int         ={
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }
         A						:     Optional[Any]         =model(SCREAMING_SNAKE_CASE__     )
         self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.seq_length, self.vocab_size)     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]      ,	SCREAMING_SNAKE_CASE__    :					List[Any]      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					Any      ,	SCREAMING_SNAKE_CASE__    :					Tuple      ,	SCREAMING_SNAKE_CASE__    :					Any      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					List[str]     )    ->				Union[str, Any]:
         A						:     str         =self.num_labels
         A						:     str         =TFRoFormerForSequenceClassification(config=SCREAMING_SNAKE_CASE__     )
         A						:     Optional[Any]         ={
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }

         A						:     Any         =model(SCREAMING_SNAKE_CASE__     )
         self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.num_labels)     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					int     )    ->				Tuple:
         A						:     str         =self.num_choices
         A						:     Dict         =TFRoFormerForMultipleChoice(config=SCREAMING_SNAKE_CASE__     )
         A						:     List[Any]         =tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE__      ,	1     )      ,	(1, self.num_choices, 1)     )
         A						:     List[str]         =tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE__      ,	1     )      ,	(1, self.num_choices, 1)     )
         A						:     Optional[int]         =tf.tile(tf.expand_dims(SCREAMING_SNAKE_CASE__      ,	1     )      ,	(1, self.num_choices, 1)     )
         A						:     Optional[int]         ={
             'input_ids': multiple_choice_inputs_ids,
             'attention_mask': multiple_choice_input_mask,
             'token_type_ids': multiple_choice_token_type_ids,
         }
         A						:     int         =model(SCREAMING_SNAKE_CASE__     )
         self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.num_choices)     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					int      ,	SCREAMING_SNAKE_CASE__    :					List[str]      ,	SCREAMING_SNAKE_CASE__    :					Any      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					List[str]     )    ->				Tuple:
         A						:     List[Any]         =self.num_labels
         A						:     Optional[int]         =TFRoFormerForTokenClassification(config=SCREAMING_SNAKE_CASE__     )
         A						:     Union[str, Any]         ={
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }
         A						:     Optional[int]         =model(SCREAMING_SNAKE_CASE__     )
         self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.seq_length, self.num_labels)     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					Any      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					Union[str, Any]     )    ->				List[str]:
         A						:     Tuple         =TFRoFormerForQuestionAnswering(config=SCREAMING_SNAKE_CASE__     )
         A						:     Dict         ={
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }

         A						:     Any         =model(SCREAMING_SNAKE_CASE__     )
         self.parent.assertEqual(result.start_logits.shape      ,	(self.batch_size, self.seq_length)     )
         self.parent.assertEqual(result.end_logits.shape      ,	(self.batch_size, self.seq_length)     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				Union[str, Any]:
         A						:     List[str]         =self.prepare_config_and_inputs()
         (
             (
             A
         )  ,  (
             A
         )  ,  (
             A
         )  ,  (
             A
         )  ,  (
             A
         )  ,  (
             A
         )  ,  (
             A
         )  ,  
         )						:     List[str]         =config_and_inputs
         A						:     Union[str, Any]         ={'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
         return config, inputs_dict







@require_tf
class  SCREAMING_SNAKE_CASE_ (					lowerCAmelCase_   ,						lowerCAmelCase_   ,						unittest.TestCase			):
  '''simple docstring'''


  lowercase		:     str            =				(
      (
          TFRoFormerModel,
          TFRoFormerForCausalLM,
          TFRoFormerForMaskedLM,
          TFRoFormerForQuestionAnswering,
          TFRoFormerForSequenceClassification,
          TFRoFormerForTokenClassification,
          TFRoFormerForMultipleChoice,
      )
      if is_tf_available()
      else ()
  )
  lowercase		:     List[Any]            =				(
      {
          "feature-extraction": TFRoFormerModel,
          "fill-mask": TFRoFormerForMaskedLM,
          "question-answering": TFRoFormerForQuestionAnswering,
          "text-classification": TFRoFormerForSequenceClassification,
          "text-generation": TFRoFormerForCausalLM,
          "token-classification": TFRoFormerForTokenClassification,
          "zero-shot": TFRoFormerForSequenceClassification,
      }
      if is_tf_available()
      else {}
  )

  lowercase		:     List[Any]            =				False
  lowercase		:     Optional[int]            =				False







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]      ,	SCREAMING_SNAKE_CASE__    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					List[str]      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					str     )    ->				Union[str, Any]:
         if pipeline_test_casse_name == "TextGenerationPipelineTests":
                return True

         return False







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				List[Any]:
         A						:     Any         =TFRoFormerModelTester(self     )
         A						:     int         =ConfigTester(self      ,	config_class=SCREAMING_SNAKE_CASE__      ,	hidden_size=37     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]     )    ->				str:
         self.config_tester.run_common_tests()







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Any     )    ->				int:
         A						:     List[Any]         =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				str:
         A						:     List[str]         =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					int     )    ->				Tuple:
         A						:     Optional[int]         =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_lm_head(*SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					int     )    ->				int:
         A						:     Tuple         =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]     )    ->				List[Any]:
         A						:     Optional[Any]         =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				List[Any]:
         A						:     Dict         =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				Any:
         A						:     List[str]         =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE__     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[int]     )    ->				Optional[int]:
         A						:     Optional[Any]         =TFRoFormerModel.from_pretrained('junnyu/roformer_chinese_base'     )
         self.assertIsNotNone(SCREAMING_SNAKE_CASE__     )







@require_tf
class  SCREAMING_SNAKE_CASE_ (					unittest.TestCase			):
  '''simple docstring'''


  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Any     )    ->				Any:
         A						:     Dict         =TFRoFormerForMaskedLM.from_pretrained('junnyu/roformer_chinese_base'     )
         A						:     Any         =tf.constant([[0, 1, 2, 3, 4, 5]]     )
         A						:     str         =model(SCREAMING_SNAKE_CASE__     )[0]

         # TODO Replace vocab size
         A						:     List[str]         =5_00_00

         A						:     Dict         =[1, 6, vocab_size]
         self.assertEqual(output.shape      ,	SCREAMING_SNAKE_CASE__     )

         print(output[:, :3, :3]     )

         # TODO Replace values below with what was printed above.
         A						:     Optional[int]         =tf.constant(
             [
                 [
                     [-0.1_2_0_5_3_3_4_1, -1.0_2_6_4_9_0_1, 0.2_9_2_2_1_9_4_6],
                     [-1.5_1_3_3_7_8_3, 0.1_9_7_4_3_3, 0.1_5_1_9_0_6_0_7],
                     [-5.0_1_3_5_4_0_3, -3.9_0_0_2_5_6, -0.8_4_0_3_8_7_6_4],
                 ]
             ]     )
         tf.debugging.assert_near(output[:, :3, :3]      ,	SCREAMING_SNAKE_CASE__      ,	atol=1e-4     )







@require_tf
class  SCREAMING_SNAKE_CASE_ (					unittest.TestCase			):
  '''simple docstring'''


  lowercase		:     Optional[Any]            =				1e-4







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]     )    ->				Any:
         A						:     Dict         =tf.constant([[4, 10]]     )
         A						:     str         =TFRoFormerSinusoidalPositionalEmbedding(num_positions=6      ,	embedding_dim=6     )

         A						:     Dict         =emba(input_ids.shape     )
         A						:     Optional[Any]         =tf.constant(
             [[0.0_0_0_0, 0.0_0_0_0, 0.0_0_0_0, 1.0_0_0_0, 1.0_0_0_0, 1.0_0_0_0], [0.8_4_1_5, 0.0_4_6_4, 0.0_0_2_2, 0.5_4_0_3, 0.9_9_8_9, 1.0_0_0_0]]     )

         tf.debugging.assert_near(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	atol=self.tolerance     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					str     )    ->				List[Any]:
         A						:     Tuple         =tf.constant(
             [
                 [0.0_0_0_0, 0.0_0_0_0, 0.0_0_0_0, 0.0_0_0_0, 0.0_0_0_0],
                 [0.8_4_1_5, 0.8_2_1_9, 0.8_0_2_0, 0.7_8_1_9, 0.7_6_1_7],
                 [0.9_0_9_3, 0.9_3_6_4, 0.9_5_8_1, 0.9_7_4_9, 0.9_8_7_0],
             ]     )
         A						:     List[str]         =TFRoFormerSinusoidalPositionalEmbedding(num_positions=5_12      ,	embedding_dim=5_12     )
         emba([2, 16, 5_12]     )
         A						:     str         =emba.weight[:3, :5]

         tf.debugging.assert_near(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	atol=self.tolerance     )







@require_tf
class  SCREAMING_SNAKE_CASE_ (					unittest.TestCase			):
  '''simple docstring'''


  lowercase		:     List[str]            =				1e-4







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Any     )    ->				Any:
         # 2,12,16,64
         A						:     List[Any]         =tf.reshape(tf.range(2 * 12 * 16 * 64      ,	dtype=tf.floataa     )      ,	shape=(2, 12, 16, 64)     ) / 1_00

         A						:     Dict         =-tf.reshape(tf.range(2 * 12 * 16 * 64      ,	dtype=tf.floataa     )      ,	shape=(2, 12, 16, 64)     ) / 1_00

         A						:     List[str]         =TFRoFormerSinusoidalPositionalEmbedding(num_positions=32      ,	embedding_dim=64     )
         A						:     Any         =embed_positions([2, 16, 7_68]     )[None, None, :, :]

         A  ,  A						:     Dict         =TFRoFormerSelfAttention.apply_rotary_position_embeddings(
             SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

         A						:     Any         =tf.constant(
             [
                 [0.0_0_0_0, 0.0_1_0_0, 0.0_2_0_0, 0.0_3_0_0, 0.0_4_0_0, 0.0_5_0_0, 0.0_6_0_0, 0.0_7_0_0],
                 [-0.2_0_1_2, 0.8_8_9_7, 0.0_2_6_3, 0.9_4_0_1, 0.2_0_7_4, 0.9_4_6_3, 0.3_4_8_1, 0.9_3_4_3],
                 [-1.7_0_5_7, 0.6_2_7_1, -1.2_1_4_5, 1.3_8_9_7, -0.6_3_0_3, 1.7_6_4_7, -0.1_1_7_3, 1.8_9_8_5],
                 [-2.1_7_3_1, -1.6_3_9_7, -2.7_3_5_8, 0.2_8_5_4, -2.1_8_4_0, 1.7_1_8_3, -1.3_0_1_8, 2.4_8_7_1],
                 [0.2_7_1_7, -3.6_1_7_3, -2.9_2_0_6, -2.1_9_8_8, -3.6_6_3_8, 0.3_8_5_8, -2.9_1_5_5, 2.2_9_8_0],
                 [3.9_8_5_9, -2.1_5_8_0, -0.7_9_8_4, -4.4_9_0_4, -4.1_1_8_1, -2.0_2_5_2, -4.4_7_8_2, 1.1_2_5_3],
             ]     )
         A						:     Union[str, Any]         =tf.constant(
             [
                 [0.0_0_0_0, -0.0_1_0_0, -0.0_2_0_0, -0.0_3_0_0, -0.0_4_0_0, -0.0_5_0_0, -0.0_6_0_0, -0.0_7_0_0],
                 [0.2_0_1_2, -0.8_8_9_7, -0.0_2_6_3, -0.9_4_0_1, -0.2_0_7_4, -0.9_4_6_3, -0.3_4_8_1, -0.9_3_4_3],
                 [1.7_0_5_7, -0.6_2_7_1, 1.2_1_4_5, -1.3_8_9_7, 0.6_3_0_3, -1.7_6_4_7, 0.1_1_7_3, -1.8_9_8_5],
                 [2.1_7_3_1, 1.6_3_9_7, 2.7_3_5_8, -0.2_8_5_4, 2.1_8_4_0, -1.7_1_8_3, 1.3_0_1_8, -2.4_8_7_1],
                 [-0.2_7_1_7, 3.6_1_7_3, 2.9_2_0_6, 2.1_9_8_8, 3.6_6_3_8, -0.3_8_5_8, 2.9_1_5_5, -2.2_9_8_0],
                 [-3.9_8_5_9, 2.1_5_8_0, 0.7_9_8_4, 4.4_9_0_4, 4.1_1_8_1, 2.0_2_5_2, 4.4_7_8_2, -1.1_2_5_3],
             ]     )

         tf.debugging.assert_near(query_layer[0, 0, :6, :8]      ,	SCREAMING_SNAKE_CASE__      ,	atol=self.tolerance     )
         tf.debugging.assert_near(key_layer[0, 0, :6, :8]      ,	SCREAMING_SNAKE_CASE__      ,	atol=self.tolerance     )







import collections
import importlib.util
import os
import re
from pathlib import Path


_lowercase							:      List[str]      							='''src/transformers'''


# Matches is_xxx_available()
_lowercase							:      Dict      							=re.compile(R'''is\_([a-z_]*)_available()''')
# Catches a one-line _import_struct = {xxx}
_lowercase							:      List[Any]      							=re.compile(R'''^_import_structure\s+=\s+\{([^\}]+)\}''')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
_lowercase							:      Tuple      							=re.compile(R'''\s+"\S*":\s+\[([^\]]*)\]''')
# Catches a line if not is_foo_available
_lowercase							:      Dict      							=re.compile(R'''^\s*if\s+not\s+is\_[a-z_]*\_available\(\)''')
# Catches a line _import_struct["bla"].append("foo")
_lowercase							:      List[Any]      							=re.compile(R'''^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)''')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
_lowercase							:      str      							=re.compile(R'''^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]''')
# Catches a line with an object between quotes and a comma:     "MyModel",
_lowercase							:      Optional[int]      							=re.compile('''^\s+"([^"]+)",''')
# Catches a line with objects between brackets only:    ["foo", "bar"],
_lowercase							:      Any      							=re.compile('''^\s+\[([^\]]+)\]''')
# Catches a line with from foo import bar, bla, boo
_lowercase							:      List[Any]      							=re.compile(R'''\s+from\s+\S*\s+import\s+([^\(\s].*)\n''')
# Catches a line with try:
_lowercase							:      Optional[Any]      							=re.compile(R'''^\s*try:''')
# Catches a line with else:
_lowercase							:      List[Any]      							=re.compile(R'''^\s*else:''')



def A__  (			lowercase:     Dict					)				->		int:
       if _re_test_backend.search(lowercase					) is None:
              return None
       A						:     Any         =[b[0] for b in _re_backend.findall(lowercase					)]
       backends.sort()
       return "_and_".join(lowercase					)



def A__  (			lowercase:     Any					)				->		List[Any]:
       with open(lowercase,		'r',		encoding='utf-8',		newline='\n'					) as f:
              A						:     Optional[Any]         =f.readlines()

       A						:     Dict         =0
       while line_index < len(lowercase					) and not lines[line_index].startswith('_import_structure = {'					):
              line_index += 1

       # If this is a traditional init, just return.
       if line_index >= len(lowercase					):
              return None

       # First grab the objects without a specific backend in _import_structure
       A						:     Optional[int]         =[]
       while not lines[line_index].startswith('if TYPE_CHECKING'					) and find_backend(lines[line_index]					) is None:
              A						:     int         =lines[line_index]
              # If we have everything on a single line, let's deal with it.
              if _re_one_line_import_struct.search(lowercase					):
                     A						:     int         =_re_one_line_import_struct.search(lowercase					).groups()[0]
                     A						:     int         =re.findall('\[([^\]]+)\]',		lowercase					)
                     for imp in imports:
                            objects.extend([obj[1:-1] for obj in imp.split(', '					)]					)
                     line_index += 1
                     continue
              A						:     Optional[int]         =_re_import_struct_key_value.search(lowercase					)
              if single_line_import_search is not None:
                     A						:     Dict         =[obj[1:-1] for obj in single_line_import_search.groups()[0].split(', '					) if len(lowercase					) > 0]
                     objects.extend(lowercase					)
              elif line.startswith(' ' * 8 + '"'					):
                     objects.append(line[9:-3]					)
              line_index += 1

       A						:     str         ={'none': objects}
       # Let's continue with backend-specific objects in _import_structure
       while not lines[line_index].startswith('if TYPE_CHECKING'					):
              # If the line is an if not is_backend_available, we grab all objects associated.
              A						:     Optional[int]         =find_backend(lines[line_index]					)
              # Check if the backend declaration is inside a try block:
              if _re_try.search(lines[line_index - 1]					) is None:
                     A						:     str         =None

              if backend is not None:
                     line_index += 1

                     # Scroll until we hit the else block of try-except-else
                     while _re_else.search(lines[line_index]					) is None:
                            line_index += 1

                     line_index += 1

                     A						:     List[str]         =[]
                     # Until we unindent, add backend objects to the list
                     while len(lines[line_index]					) <= 1 or lines[line_index].startswith(' ' * 4					):
                            A						:     Optional[Any]         =lines[line_index]
                            if _re_import_struct_add_one.search(lowercase					) is not None:
                                   objects.append(_re_import_struct_add_one.search(lowercase					).groups()[0]					)
                            elif _re_import_struct_add_many.search(lowercase					) is not None:
                                   A						:     Optional[Any]         =_re_import_struct_add_many.search(lowercase					).groups()[0].split(', '					)
                                   A						:     int         =[obj[1:-1] for obj in imports if len(lowercase					) > 0]
                                   objects.extend(lowercase					)
                            elif _re_between_brackets.search(lowercase					) is not None:
                                   A						:     Optional[int]         =_re_between_brackets.search(lowercase					).groups()[0].split(', '					)
                                   A						:     Optional[int]         =[obj[1:-1] for obj in imports if len(lowercase					) > 0]
                                   objects.extend(lowercase					)
                            elif _re_quote_object.search(lowercase					) is not None:
                                   objects.append(_re_quote_object.search(lowercase					).groups()[0]					)
                            elif line.startswith(' ' * 8 + '"'					):
                                   objects.append(line[9:-3]					)
                            elif line.startswith(' ' * 12 + '"'					):
                                   objects.append(line[13:-3]					)
                            line_index += 1

                     A						:     Optional[Any]         =objects
              else:
                     line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
       A						:     Optional[Any]         =[]
       while (
           line_index < len(lowercase					)
           and find_backend(lines[line_index]					) is None
           and not lines[line_index].startswith('else'					)
       ):
              A						:     Any         =lines[line_index]
              A						:     Optional[int]         =_re_import.search(lowercase					)
              if single_line_import_search is not None:
                     objects.extend(single_line_import_search.groups()[0].split(', '					)					)
              elif line.startswith(' ' * 8					):
                     objects.append(line[8:-2]					)
              line_index += 1

       A						:     Optional[Any]         ={'none': objects}
       # Let's continue with backend-specific objects
       while line_index < len(lowercase					):
              # If the line is an if is_backend_available, we grab all objects associated.
              A						:     str         =find_backend(lines[line_index]					)
              # Check if the backend declaration is inside a try block:
              if _re_try.search(lines[line_index - 1]					) is None:
                     A						:     Optional[Any]         =None

              if backend is not None:
                     line_index += 1

                     # Scroll until we hit the else block of try-except-else
                     while _re_else.search(lines[line_index]					) is None:
                            line_index += 1

                     line_index += 1

                     A						:     List[str]         =[]
                     # Until we unindent, add backend objects to the list
                     while len(lines[line_index]					) <= 1 or lines[line_index].startswith(' ' * 8					):
                            A						:     Any         =lines[line_index]
                            A						:     Any         =_re_import.search(lowercase					)
                            if single_line_import_search is not None:
                                   objects.extend(single_line_import_search.groups()[0].split(', '					)					)
                            elif line.startswith(' ' * 12					):
                                   objects.append(line[12:-2]					)
                            line_index += 1

                     A						:     Dict         =objects
              else:
                     line_index += 1

       return import_dict_objects, type_hint_objects



def A__  (			lowercase:     Any,		lowercase:     int					)				->		Dict:

       def find_duplicates(lowercase:     List[str]					):
              return [k for k, v in collections.Counter(lowercase					).items() if v > 1]

       if list(import_dict_objects.keys()					) != list(type_hint_objects.keys()					):
              return ["Both sides of the init do not have the same backends!"]

       A						:     List[Any]         =[]
       for key in import_dict_objects.keys():
              A						:     List[Any]         =find_duplicates(import_dict_objects[key]					)
              if duplicate_imports:
                     errors.append(F'Duplicate _import_structure definitions for: {duplicate_imports}'					)
              A						:     Tuple         =find_duplicates(type_hint_objects[key]					)
              if duplicate_type_hints:
                     errors.append(F'Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'					)

              if sorted(set(import_dict_objects[key]					)					) != sorted(set(type_hint_objects[key]					)					):
                     A						:     Tuple         ='base imports' if key == 'none' else F'{key} backend'
                     errors.append(F'Differences for {name}:'					)
                     for a in type_hint_objects[key]:
                            if a not in import_dict_objects[key]:
                                   errors.append(F'  {a} in TYPE_HINT but not in _import_structure.'					)
                     for a in import_dict_objects[key]:
                            if a not in type_hint_objects[key]:
                                   errors.append(F'  {a} in _import_structure but not in TYPE_HINT.'					)
       return errors



def A__  (			)				->		List[str]:
       A						:     Dict         =[]
       for root, _, files in os.walk(lowercase					):
              if "__init__.py" in files:
                     A						:     Any         =os.path.join(lowercase,		'__init__.py'					)
                     A						:     Union[str, Any]         =parse_init(lowercase					)
                     if objects is not None:
                            A						:     str         =analyze_results(*lowercase					)
                            if len(lowercase					) > 0:
                                   A						:     Any         =F'Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'
                                   failures.append('\n'.join(lowercase					)					)
       if len(lowercase					) > 0:
              raise ValueError('\n\n'.join(lowercase					)					)



def A__  (			)				->		int:
       A						:     List[str]         =[]
       for path, directories, files in os.walk(lowercase					):
              for folder in directories:
                     # Ignore private modules
                     if folder.startswith('_'					):
                            directories.remove(lowercase					)
                            continue
                     # Ignore leftovers from branches (empty folders apart from pycache)
                     if len(list((Path(lowercase					) / folder).glob('*.py'					)					)					) == 0:
                            continue
                     A						:     Any         =str((Path(lowercase					) / folder).relative_to(lowercase					)					)
                     A						:     List[str]         =short_path.replace(os.path.sep,		'.'					)
                     submodules.append(lowercase					)
              for fname in files:
                     if fname == "__init__.py":
                            continue
                     A						:     Optional[Any]         =str((Path(lowercase					) / fname).relative_to(lowercase					)					)
                     A						:     Dict         =short_path.replace('.py',		''					).replace(os.path.sep,		'.'					)
                     if len(submodule.split('.'					)					) == 1:
                            submodules.append(lowercase					)
       return submodules


_lowercase							:      Tuple      							=[
    '''convert_pytorch_checkpoint_to_tf2''',
    '''modeling_flax_pytorch_utils''',
]



def A__  (			)				->		Tuple:
       # This is to make sure the transformers module imported is the one in the repo.
       A						:     str         =importlib.util.spec_from_file_location(
           'transformers',		os.path.join(lowercase,		'__init__.py'					),		submodule_search_locations=[PATH_TO_TRANSFORMERS],		)
       A						:     Any         =spec.loader.load_module()

       A						:     Any         =[
           module
           for module in get_transformers_submodules()
           if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
       ]
       if len(lowercase					) > 0:
              A						:     Dict         ='\n'.join(F'- {module}' for module in module_not_registered					)
              raise ValueError(
                  'The following submodules are not properly registered in the main init of Transformers:\n'
                  F'{list_of_modules}\n'
                  'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'					)


if __name__ == "__main__":
    check_all_inits()
    check_submodules()






