






"""simple docstring"""





import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)





def        a_  (   _lowerCAmelCase						:				int  ):





       '''simple docstring'''

       if model.config.model_type == "gpt2":
              return model.transformer.h[0].mlp.c_fc
       return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
       import torch
       import torch.nn as nn


       class   UpperCAmelCase_			(					nn.Module):


             def __init__(     self	,    a	,    a							) ->      int:
                    super().__init__()
                    lowercase__    :	Any     					=			module
                    lowercase__    :	int     					=			nn.Sequential(
                        nn.Linear(module.in_features	,    _lowerCamelCase	,    bias=_lowerCamelCase							)	,    nn.Linear(_lowerCamelCase	,    module.out_features	,    bias=_lowerCamelCase							)	,    )
                    lowercase__    :	Dict     					=			(2.0 / (5 * min(module.in_features	,    module.out_features							))) ** 0.5
                    nn.init.normal_(self.adapter[0].weight	,    std=_lowerCamelCase							)
                    nn.init.zeros_(self.adapter[1].weight							)
                    self.adapter.to(module.weight.device							)


             def 	_UpperCAmelCase (     self	,    a	,    *a	,    **a							) ->      Dict:
                    return self.module(_lowerCamelCase	,    *_lowerCamelCase	,    **_lowerCamelCase							) + self.adapter(_lowerCamelCase							)


@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class   UpperCAmelCase_			(					unittest.TestCase):
      lowerCamelCase__	:					Any										=						"bigscience/bloom-1b7"

      # Constant values
      lowerCamelCase__	:					str										=						2.109659552692574

      lowerCamelCase__	:					Tuple										=						"Hello my name is"
      lowerCamelCase__	:					Any										=						set()
      EXPECTED_OUTPUTS.add("Hello my name is John and I am a professional photographer. I")
      EXPECTED_OUTPUTS.add("Hello my name is John.\nI am a friend of your father.\n")
      EXPECTED_OUTPUTS.add("Hello my name is John Doe, I am a student at the University")
      lowerCamelCase__	:					List[str]										=						1_0


      def 	_UpperCAmelCase (     self							) ->      List[str]:
             lowercase__    :	List[str]     					=			AutoTokenizer.from_pretrained(self.model_name							)





class   UpperCAmelCase_			(					lowerCAmelCase__):


      def 	_UpperCAmelCase (     self							) ->      str:
             super().setUp()

             # Models and tokenizer
             lowercase__    :	Dict     					=			AutoModelForCausalLM.from_pretrained(
                 self.model_name	,    torch_dtype=torch.floataa	,    device_map='auto'							)
             lowercase__    :	List[Any]     					=			AutoModelForCausalLM.from_pretrained(self.model_name	,    load_in_abit=_lowerCamelCase	,    device_map='auto'							)


      def 	_UpperCAmelCase (     self							) ->      int:
             del self.model_fpaa
             del self.model_abit

             gc.collect()
             torch.cuda.empty_cache()


      def 	_UpperCAmelCase (     self							) ->      str:
             lowercase__    :	List[Any]     					=			self.model_abit.config

             self.assertTrue(hasattr(_lowerCamelCase	,    'quantization_config'							)							)

             lowercase__    :	Optional[Any]     					=			config.to_dict()
             lowercase__    :	Optional[Any]     					=			config.to_diff_dict()

             lowercase__    :	Any     					=			config.to_json_string()


      def 	_UpperCAmelCase (     self							) ->      Union[str, Any]:
             from bitsandbytes.nn import Paramsabit

             lowercase__    :	Tuple     					=			self.model_fpaa.get_memory_footprint()
             lowercase__    :	Any     					=			self.model_abit.get_memory_footprint()

             self.assertAlmostEqual(mem_fpaa / mem_abit	,    self.EXPECTED_RELATIVE_DIFFERENCE							)
             lowercase__    :	Any     					=			get_some_linear_layer(self.model_abit							)
             self.assertTrue(linear.weight.__class__ == Paramsabit							)


      def 	_UpperCAmelCase (     self							) ->      Tuple:
             from transformers import TaPreTrainedModel

             self.model_fpaa.get_memory_footprint()
             self.model_abit.get_memory_footprint()

             for name, module in self.model_abit.named_modules():
                    if isinstance(_lowerCamelCase	,    torch.nn.Linear							):
                           if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
                                  # 4-bit parameters are packed in uint8 variables
                                  self.assertTrue(module.weight.dtype == torch.uinta							)


      def 	_UpperCAmelCase (     self							) ->      Tuple:
             lowercase__    :	List[str]     					=			self.tokenizer(self.input_text	,    return_tensors='pt'							)
             lowercase__    :	Dict     					=			self.model_abit.generate(input_ids=encoded_input['input_ids'].to(0							)	,    max_new_tokens=1_0							)

             self.assertIn(self.tokenizer.decode(output_sequences[0]	,    skip_special_tokens=_lowerCamelCase							)	,    self.EXPECTED_OUTPUTS							)


      def 	_UpperCAmelCase (     self							) ->      Tuple:
             lowercase__    :	Optional[Any]     					=			BitsAndBytesConfig()
             lowercase__    :	Any     					=			True

             lowercase__    :	str     					=			AutoModelForCausalLM.from_pretrained(
                 self.model_name	,    quantization_config=_lowerCamelCase	,    device_map='auto'							)

             lowercase__    :	Dict     					=			self.tokenizer(self.input_text	,    return_tensors='pt'							)
             lowercase__    :	Optional[int]     					=			model_abit_from_config.generate(
                 input_ids=encoded_input['input_ids'].to(0							)	,    max_new_tokens=1_0							)

             self.assertIn(self.tokenizer.decode(output_sequences[0]	,    skip_special_tokens=_lowerCamelCase							)	,    self.EXPECTED_OUTPUTS							)


      def 	_UpperCAmelCase (     self							) ->      Any:





             with self.assertRaises(_lowerCamelCase							), tempfile.TemporaryDirectory() as tmpdirname:
                    self.model_abit.save_pretrained(_lowerCamelCase							)


      def 	_UpperCAmelCase (     self							) ->      Dict:
             lowercase__    :	Optional[Any]     					=			BitsAndBytesConfig()

             with self.assertRaises(_lowerCamelCase							):
                    lowercase__    :	Optional[Any]     					=			AutoModelForCausalLM.from_pretrained(
                        self.model_name	,    quantization_config=_lowerCamelCase	,    load_in_abit=_lowerCamelCase	,    device_map='auto'	,    bnb_abit_quant_type='nf4'	,    )


      def 	_UpperCAmelCase (     self							) ->      Optional[int]:





             with self.assertRaises(_lowerCamelCase							):
                    # Tries with `str`
                    self.model_abit.to('cpu'							)

             with self.assertRaises(_lowerCamelCase							):
                    # Tries with a `dtype``
                    self.model_abit.to(torch.floataa							)

             with self.assertRaises(_lowerCamelCase							):
                    # Tries with a `device`
                    self.model_abit.to(torch.device('cuda:0'							)							)

             with self.assertRaises(_lowerCamelCase							):
                    # Tries with a `device`
                    self.model_abit.float()

             with self.assertRaises(_lowerCamelCase							):
                    # Tries with a `device`
                    self.model_abit.half()

             # Test if we did not break anything
             lowercase__    :	List[str]     					=			self.tokenizer(self.input_text	,    return_tensors='pt'							)

             lowercase__    :	Union[str, Any]     					=			self.model_fpaa.to(torch.floataa							)
             lowercase__    :	Tuple     					=			self.model_fpaa.generate(input_ids=encoded_input['input_ids'].to(0							)	,    max_new_tokens=1_0							)

             # Check this does not throw an error
             lowercase__    :	Union[str, Any]     					=			self.model_fpaa.to('cpu'							)

             # Check this does not throw an error
             lowercase__    :	Optional[Any]     					=			self.model_fpaa.half()

             # Check this does not throw an error
             lowercase__    :	Dict     					=			self.model_fpaa.float()


      def 	_UpperCAmelCase (     self							) ->      Optional[Any]:
             lowercase__    :	List[Any]     					=			AutoModelForSeqaSeqLM.from_pretrained('t5-small'	,    load_in_abit=_lowerCamelCase	,    device_map='auto'							)
             self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa							)





@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class   UpperCAmelCase_			(					unittest.TestCase):


      @classmethod
      def 	_UpperCAmelCase (     cls							) ->      List[Any]:
             lowercase__    :	List[str]     					=			't5-small'
             lowercase__    :	Union[str, Any]     					=			'google/flan-t5-small'  # flan-t5 uses dense-act instead of dense-relu-dense
             lowercase__    :	Any     					=			AutoTokenizer.from_pretrained(cls.model_name							)
             lowercase__    :	Tuple     					=			'Translate in German: Hello, my dog is cute'


      def 	_UpperCAmelCase (     self							) ->      str:
             gc.collect()
             torch.cuda.empty_cache()


      def 	_UpperCAmelCase (     self							) ->      List[Any]:
             from transformers import TaForConditionalGeneration

             lowercase__    :	Dict     					=			TaForConditionalGeneration._keep_in_fpaa_modules
             lowercase__    :	Tuple     					=			None

             # test with `t5-small`
             lowercase__    :	Tuple     					=			TaForConditionalGeneration.from_pretrained(self.model_name	,    load_in_abit=_lowerCamelCase	,    device_map='auto'							)
             lowercase__    :	List[Any]     					=			self.tokenizer(self.input_text	,    return_tensors='pt'							).to(0							)
             lowercase__    :	Any     					=			model.generate(**_lowerCamelCase							)

             # test with `flan-t5-small`
             lowercase__    :	Dict     					=			TaForConditionalGeneration.from_pretrained(
                 self.dense_act_model_name	,    load_in_abit=_lowerCamelCase	,    device_map='auto'							)
             lowercase__    :	Optional[int]     					=			self.tokenizer(self.input_text	,    return_tensors='pt'							).to(0							)
             lowercase__    :	Tuple     					=			model.generate(**_lowerCamelCase							)
             lowercase__    :	str     					=			modules


      def 	_UpperCAmelCase (     self							) ->      List[Any]:
             import bitsandbytes as bnb

             from transformers import TaForConditionalGeneration

             # test with `t5-small`
             lowercase__    :	List[Any]     					=			TaForConditionalGeneration.from_pretrained(self.model_name	,    load_in_abit=_lowerCamelCase	,    device_map='auto'							)

             # there was a bug with decoders - this test checks that it is fixed
             self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q	,    bnb.nn.Linearabit							)							)

             lowercase__    :	Tuple     					=			self.tokenizer(self.input_text	,    return_tensors='pt'							).to(0							)
             lowercase__    :	str     					=			model.generate(**_lowerCamelCase							)

             # test with `flan-t5-small`
             lowercase__    :	Optional[int]     					=			TaForConditionalGeneration.from_pretrained(
                 self.dense_act_model_name	,    load_in_abit=_lowerCamelCase	,    device_map='auto'							)
             lowercase__    :	int     					=			self.tokenizer(self.input_text	,    return_tensors='pt'							).to(0							)
             lowercase__    :	Union[str, Any]     					=			model.generate(**_lowerCamelCase							)





class   UpperCAmelCase_			(					lowerCAmelCase__):


      def 	_UpperCAmelCase (     self							) ->      Union[str, Any]:
             super().setUp()
             # model_name
             lowercase__    :	Optional[Any]     					=			'bigscience/bloom-560m'
             lowercase__    :	Optional[int]     					=			't5-small'

             # Different types of model

             lowercase__    :	Optional[Any]     					=			AutoModel.from_pretrained(self.model_name	,    load_in_abit=_lowerCamelCase	,    device_map='auto'							)
             # Sequence classification model
             lowercase__    :	List[Any]     					=			AutoModelForSequenceClassification.from_pretrained(
                 self.model_name	,    load_in_abit=_lowerCamelCase	,    device_map='auto'							)
             # CausalLM model
             lowercase__    :	List[str]     					=			AutoModelForCausalLM.from_pretrained(self.model_name	,    load_in_abit=_lowerCamelCase	,    device_map='auto'							)
             # Seq2seq model
             lowercase__    :	Union[str, Any]     					=			AutoModelForSeqaSeqLM.from_pretrained(
                 self.seq_to_seq_name	,    load_in_abit=_lowerCamelCase	,    device_map='auto'							)


      def 	_UpperCAmelCase (     self							) ->      Union[str, Any]:
             del self.base_model
             del self.sequence_model
             del self.model_abit
             del self.seq_to_seq_model

             gc.collect()
             torch.cuda.empty_cache()


      def 	_UpperCAmelCase (     self							) ->      str:
             from bitsandbytes.nn import Paramsabit

             self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit							)

             # Other heads should be nn.Parameter
             self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter							)
             self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter							)
             self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter							)





class   UpperCAmelCase_			(					lowerCAmelCase__):


      def 	_UpperCAmelCase (     self							) ->      Tuple:
             super().setUp()


      def 	_UpperCAmelCase (     self							) ->      Dict:
             del self.pipe

             gc.collect()
             torch.cuda.empty_cache()


      def 	_UpperCAmelCase (     self							) ->      Union[str, Any]:
             lowercase__    :	Any     					=			pipeline(
                 'text-generation'	,    model=self.model_name	,    model_kwargs={'device_map': 'auto', 'load_in_4bit': True, 'torch_dtype': torch.floataa}	,    max_new_tokens=self.MAX_NEW_TOKENS	,    )

             # Real second forward pass
             lowercase__    :	Optional[Any]     					=			self.pipe(self.input_text							)
             self.assertIn(pipeline_output[0]['generated_text']	,    self.EXPECTED_OUTPUTS							)





@require_torch_multi_gpu
class   UpperCAmelCase_			(					lowerCAmelCase__):


      def 	_UpperCAmelCase (     self							) ->      Optional[Any]:
             super().setUp()


      def 	_UpperCAmelCase (     self							) ->      Optional[Any]:
             lowercase__    :	Union[str, Any]     					=			AutoModelForCausalLM.from_pretrained(
                 self.model_name	,    load_in_abit=_lowerCamelCase	,    device_map='balanced'							)

             # Check correct device map
             self.assertEqual(set(model_parallel.hf_device_map.values()							)	,    {0, 1}							)

             # Check that inference pass works on the model
             lowercase__    :	Any     					=			self.tokenizer(self.input_text	,    return_tensors='pt'							)

             # Second real batch
             lowercase__    :	Dict     					=			model_parallel.generate(input_ids=encoded_input['input_ids'].to(0							)	,    max_new_tokens=1_0							)
             self.assertIn(self.tokenizer.decode(output_parallel[0]	,    skip_special_tokens=_lowerCamelCase							)	,    self.EXPECTED_OUTPUTS							)





class   UpperCAmelCase_			(					lowerCAmelCase__):


      def 	_UpperCAmelCase (     self							) ->      int:
             lowercase__    :	Tuple     					=			'facebook/opt-350m'
             super().setUp()


      def 	_UpperCAmelCase (     self							) ->      str:





             if version.parse(importlib.metadata.version('bitsandbytes'							)							) < version.parse('0.37.0'							):
                    return

             # Step 1: freeze all parameters
             lowercase__    :	Optional[Any]     					=			AutoModelForCausalLM.from_pretrained(self.model_name	,    load_in_abit=_lowerCamelCase							)

             self.assertEqual(set(model.hf_device_map.values()							)	,    {torch.cuda.current_device()}							)

             for param in model.parameters():
                    lowercase__    :	Optional[Any]     					=			False  # freeze the model - train adapters later
                    if param.ndim == 1:
                           # cast the small parameters (e.g. layernorm) to fp32 for stability
                           lowercase__    :	List[str]     					=			param.data.to(torch.floataa							)

        # Step 2: add adapters
             for _, module in model.named_modules():
                    if "OPTAttention" in repr(type(_lowerCamelCase							)							):
                           lowercase__    :	List[Any]     					=			LoRALayer(module.q_proj	,    rank=1_6							)
                           lowercase__    :	str     					=			LoRALayer(module.k_proj	,    rank=1_6							)
                           lowercase__    :	Optional[int]     					=			LoRALayer(module.v_proj	,    rank=1_6							)

        # Step 3: dummy batch
             lowercase__    :	Optional[int]     					=			self.tokenizer('Test batch '	,    return_tensors='pt'							).to(0							)

             # Step 4: Check if the gradient is not None
             with torch.cuda.amp.autocast():
                    lowercase__    :	List[str]     					=			model.forward(**_lowerCamelCase							)
                    out.logits.norm().backward()

             for module in model.modules():
                    if isinstance(_lowerCamelCase	,    _lowerCamelCase							):
                           self.assertTrue(module.adapter[1].weight.grad is not None							)
                           self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0							)
                    elif isinstance(_lowerCamelCase	,    nn.Embedding							):
                           self.assertTrue(module.weight.grad is None							)





class   UpperCAmelCase_			(					lowerCAmelCase__):
      lowerCamelCase__	:					Dict										=						"gpt2-xl"
      lowerCamelCase__	:					Union[str, Any]										=						3.3191854854152187


"""simple docstring"""





import os
import tempfile
import unittest

from transformers import DistilBertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
           DistilBertForMaskedLM,
           DistilBertForMultipleChoice,
           DistilBertForQuestionAnswering,
           DistilBertForSequenceClassification,
           DistilBertForTokenClassification,
           DistilBertModel,
       )


class   UpperCAmelCase_			(					_a):


      def __init__(     self	,    a	,    a=1_3	,    a=7	,    a=True	,    a=True	,    a=False	,    a=True	,    a=9_9	,    a=3_2	,    a=5	,    a=4	,    a=3_7	,    a="gelu"	,    a=0.1	,    a=0.1	,    a=5_1_2	,    a=1_6	,    a=2	,    a=0.02	,    a=3	,    a=4	,    a=None	,    ) ->      Any:
             lowercase__    :	Tuple     					=			parent
             lowercase__    :	List[Any]     					=			batch_size
             lowercase__    :	List[Any]     					=			seq_length
             lowercase__    :	List[Any]     					=			is_training
             lowercase__    :	Optional[Any]     					=			use_input_mask
             lowercase__    :	Optional[int]     					=			use_token_type_ids
             lowercase__    :	int     					=			use_labels
             lowercase__    :	Tuple     					=			vocab_size
             lowercase__    :	int     					=			hidden_size
             lowercase__    :	Any     					=			num_hidden_layers
             lowercase__    :	List[str]     					=			num_attention_heads
             lowercase__    :	Optional[Any]     					=			intermediate_size
             lowercase__    :	Optional[Any]     					=			hidden_act
             lowercase__    :	List[str]     					=			hidden_dropout_prob
             lowercase__    :	List[Any]     					=			attention_probs_dropout_prob
             lowercase__    :	List[Any]     					=			max_position_embeddings
             lowercase__    :	List[str]     					=			type_vocab_size
             lowercase__    :	Tuple     					=			type_sequence_label_size
             lowercase__    :	List[Any]     					=			initializer_range
             lowercase__    :	str     					=			num_labels
             lowercase__    :	Tuple     					=			num_choices
             lowercase__    :	str     					=			scope


      def 	_UpperCAmelCase (     self							) ->      Any:
             lowercase__    :	List[Any]     					=			ids_tensor([self.batch_size, self.seq_length]	,    self.vocab_size							)

             lowercase__    :	str     					=			None
             if self.use_input_mask:
                    lowercase__    :	Any     					=			random_attention_mask([self.batch_size, self.seq_length]							)

             lowercase__    :	Dict     					=			None
             lowercase__    :	Optional[Any]     					=			None
             lowercase__    :	int     					=			None
             if self.use_labels:
                    lowercase__    :	Union[str, Any]     					=			ids_tensor([self.batch_size]	,    self.type_sequence_label_size							)
                    lowercase__    :	List[Any]     					=			ids_tensor([self.batch_size, self.seq_length]	,    self.num_labels							)
                    lowercase__    :	Dict     					=			ids_tensor([self.batch_size]	,    self.num_choices							)

             lowercase__    :	List[Any]     					=			self.get_config()

             return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


      def 	_UpperCAmelCase (     self							) ->      Optional[int]:
             return DistilBertConfig(
                 vocab_size=self.vocab_size	,    dim=self.hidden_size	,    n_layers=self.num_hidden_layers	,    n_heads=self.num_attention_heads	,    hidden_dim=self.intermediate_size	,    hidden_act=self.hidden_act	,    dropout=self.hidden_dropout_prob	,    attention_dropout=self.attention_probs_dropout_prob	,    max_position_embeddings=self.max_position_embeddings	,    initializer_range=self.initializer_range	,    )


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    a	,    a	,    a							) ->      Dict:
             lowercase__    :	Tuple     					=			DistilBertModel(config=a							)
             model.to(a							)
             model.eval()
             lowercase__    :	Any     					=			model(a	,    a							)
             lowercase__    :	str     					=			model(a							)
             self.parent.assertEqual(result.last_hidden_state.shape	,    (self.batch_size, self.seq_length, self.hidden_size)							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    a	,    a	,    a							) ->      Dict:
             lowercase__    :	Optional[int]     					=			DistilBertForMaskedLM(config=a							)
             model.to(a							)
             model.eval()
             lowercase__    :	Union[str, Any]     					=			model(a	,    attention_mask=a	,    labels=a							)
             self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.vocab_size)							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    a	,    a	,    a							) ->      int:
             lowercase__    :	Tuple     					=			DistilBertForQuestionAnswering(config=a							)
             model.to(a							)
             model.eval()
             lowercase__    :	Tuple     					=			model(
                 a	,    attention_mask=a	,    start_positions=a	,    end_positions=a							)
             self.parent.assertEqual(result.start_logits.shape	,    (self.batch_size, self.seq_length)							)
             self.parent.assertEqual(result.end_logits.shape	,    (self.batch_size, self.seq_length)							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    a	,    a	,    a							) ->      List[str]:
             lowercase__    :	int     					=			self.num_labels
             lowercase__    :	Dict     					=			DistilBertForSequenceClassification(a							)
             model.to(a							)
             model.eval()
             lowercase__    :	Optional[Any]     					=			model(a	,    attention_mask=a	,    labels=a							)
             self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_labels)							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    a	,    a	,    a							) ->      Any:
             lowercase__    :	Any     					=			self.num_labels
             lowercase__    :	List[str]     					=			DistilBertForTokenClassification(config=a							)
             model.to(a							)
             model.eval()

             lowercase__    :	Any     					=			model(a	,    attention_mask=a	,    labels=a							)
             self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.num_labels)							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a	,    a	,    a	,    a							) ->      Tuple:
             lowercase__    :	List[Any]     					=			self.num_choices
             lowercase__    :	Any     					=			DistilBertForMultipleChoice(config=a							)
             model.to(a							)
             model.eval()
             lowercase__    :	str     					=			input_ids.unsqueeze(1							).expand(-1	,    self.num_choices	,    -1							).contiguous()
             lowercase__    :	Optional[int]     					=			input_mask.unsqueeze(1							).expand(-1	,    self.num_choices	,    -1							).contiguous()
             lowercase__    :	int     					=			model(
                 a	,    attention_mask=a	,    labels=a	,    )
             self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_choices)							)


      def 	_UpperCAmelCase (     self							) ->      str:
             lowercase__    :	Union[str, Any]     					=			self.prepare_config_and_inputs()
             ((lowercase__)					,   (lowercase__)					,   (lowercase__)					,   (lowercase__)					,   (lowercase__)					,   (lowercase__))    :	List[str]     					=			config_and_inputs
             lowercase__    :	Optional[Any]     					=			{'input_ids': input_ids, 'attention_mask': input_mask}
             return config, inputs_dict





@require_torch
class   UpperCAmelCase_			(					_a				,							_a				,							unittest.TestCase):
      lowerCamelCase__	:					List[str]										=						(
          (
              DistilBertModel,
              DistilBertForMaskedLM,
              DistilBertForMultipleChoice,
              DistilBertForQuestionAnswering,
              DistilBertForSequenceClassification,
              DistilBertForTokenClassification,
          )
          if is_torch_available()
          else None
      )
      lowerCamelCase__	:					str										=						(
          {
              "feature-extraction": DistilBertModel,
              "fill-mask": DistilBertForMaskedLM,
              "question-answering": DistilBertForQuestionAnswering,
              "text-classification": DistilBertForSequenceClassification,
              "token-classification": DistilBertForTokenClassification,
              "zero-shot": DistilBertForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      lowerCamelCase__	:					Optional[int]										=						True
      lowerCamelCase__	:					Any										=						True
      lowerCamelCase__	:					List[Any]										=						True
      lowerCamelCase__	:					Optional[Any]										=						True


      def 	_UpperCAmelCase (     self							) ->      Union[str, Any]:
             lowercase__    :	str     					=			DistilBertModelTester(self							)
             lowercase__    :	int     					=			ConfigTester(self	,    config_class=a	,    dim=3_7							)


      def 	_UpperCAmelCase (     self							) ->      Dict:
             self.config_tester.run_common_tests()


      def 	_UpperCAmelCase (     self							) ->      Optional[Any]:
             lowercase__    :	int     					=			self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_distilbert_model(*a							)


      def 	_UpperCAmelCase (     self							) ->      Any:
             lowercase__    :	int     					=			self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_distilbert_for_masked_lm(*a							)


      def 	_UpperCAmelCase (     self							) ->      Optional[int]:
             lowercase__    :	str     					=			self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_distilbert_for_question_answering(*a							)


      def 	_UpperCAmelCase (     self							) ->      int:
             lowercase__    :	List[str]     					=			self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_distilbert_for_sequence_classification(*a							)


      def 	_UpperCAmelCase (     self							) ->      List[str]:
             lowercase__    :	Dict     					=			self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_distilbert_for_token_classification(*a							)


      def 	_UpperCAmelCase (     self							) ->      str:
             lowercase__    :	Any     					=			self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_distilbert_for_multiple_choice(*a							)


      @slow
      def 	_UpperCAmelCase (     self							) ->      str:
             for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                    lowercase__    :	str     					=			DistilBertModel.from_pretrained(a							)
                    self.assertIsNotNone(a							)


      @slow
      @require_torch_gpu
      def 	_UpperCAmelCase (     self							) ->      Any:
             lowercase__					,   lowercase__    :	Dict     					=			self.model_tester.prepare_config_and_inputs_for_common()
             for model_class in self.all_model_classes:
                    # BertForMultipleChoice behaves incorrectly in JIT environments.
                    if model_class == DistilBertForMultipleChoice:
                           return

                    lowercase__    :	Optional[int]     					=			True
                    lowercase__    :	Union[str, Any]     					=			model_class(config=a							)

                    lowercase__    :	int     					=			self._prepare_for_class(a	,    a							)
                    lowercase__    :	Tuple     					=			torch.jit.trace(
                        a	,    (inputs_dict['input_ids'].to('cpu'							), inputs_dict['attention_mask'].to('cpu'							))							)

                    with tempfile.TemporaryDirectory() as tmp:
                           torch.jit.save(a	,    os.path.join(a	,    'traced_model.pt'							)							)
                           lowercase__    :	Optional[int]     					=			torch.jit.load(os.path.join(a	,    'traced_model.pt'							)	,    map_location=a							)
                           loaded(inputs_dict['input_ids'].to(a							)	,    inputs_dict['attention_mask'].to(a							)							)





@require_torch
class   UpperCAmelCase_			(					unittest.TestCase):


      @slow
      def 	_UpperCAmelCase (     self							) ->      List[str]:
             lowercase__    :	int     					=			DistilBertModel.from_pretrained('distilbert-base-uncased'							)
             lowercase__    :	Union[str, Any]     					=			torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]							)
             lowercase__    :	Optional[Any]     					=			torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]							)
             with torch.no_grad():
                    lowercase__    :	Optional[Any]     					=			model(a	,    attention_mask=a							)[0]
             lowercase__    :	Tuple     					=			torch.Size((1, 1_1, 7_6_8)							)
             self.assertEqual(output.shape	,    a							)
             lowercase__    :	List[Any]     					=			torch.tensor(
                 [[[-0.1_639, 0.3_299, 0.1_648], [-0.1_746, 0.3_289, 0.1_710], [-0.1_884, 0.3_357, 0.1_810]]]							)

             self.assertTrue(torch.allclose(output[:, 1:4, 1:4]	,    a	,    atol=1e-4							)							)

