


import math
from collections import defaultdict
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput
def      snake_case_						( __lowercase			,       __lowercase=0.9_9_9			,       __lowercase="cosine"			,       ):
   if alpha_transform_type == "cosine":

      def alpha_bar_fn(__lowercase       ):
         return math.cos((t + 0.0_0_8) / 1.0_0_8 * math.pi / 2       ) ** 2

   elif alpha_transform_type == "exp":

      def alpha_bar_fn(__lowercase       ):
         return math.exp(t * -1_2.0       )

   else:
      raise ValueError(F'''Unsupported alpha_tranform_type: {alpha_transform_type}'''       )

   UpperCAmelCase_							:      str						=   []
   for i in range(__lowercase       ):
      UpperCAmelCase_							:      str						=   i / num_diffusion_timesteps
      UpperCAmelCase_							:      List[Any]						=   (i + 1) / num_diffusion_timesteps
      betas.append(min(1 - alpha_bar_fn(__lowercase       ) / alpha_bar_fn(__lowercase       )			,       __lowercase       )       )
   return torch.tensor(__lowercase			,       dtype=torch.floataa       )





class  lowerCAmelCase__(    snake_case__    ,    snake_case__						):






    '''simple docstring'''





    A_  :     Tuple			= [e.name for e in KarrasDiffusionSchedulers]
    A_  :     int			= 2





    @register_to_config
    def __init__(	self				:   Any       ,   __snake_case				:   int = 1_000       ,   __snake_case				:   float = 0.00_085       ,   __snake_case				:   float = 0.012       ,   __snake_case				:   str = "linear"       ,   __snake_case				:   Optional[Union[np.ndarray, List[float]]] = None       ,   __snake_case				:   str = "epsilon"       ,   __snake_case				:   str = "linspace"       ,   __snake_case				:   int = 0       ,   ):





       '''simple docstring'''
       if trained_betas is not None:
          UpperCAmelCase_							:      Optional[Any]						=   torch.tensor(__snake_case       ,   dtype=torch.floataa	)
       elif beta_schedule == "linear":
          UpperCAmelCase_							:      Dict						=   torch.linspace(__snake_case       ,   __snake_case       ,   __snake_case       ,   dtype=torch.floataa	)
       elif beta_schedule == "scaled_linear":
          # this schedule is very specific to the latent diffusion model.
          UpperCAmelCase_							:      List[str]						=   (
              torch.linspace(beta_start**0.5       ,   beta_end**0.5       ,   __snake_case       ,   dtype=torch.floataa	) ** 2
          )
       elif beta_schedule == "squaredcos_cap_v2":
          # Glide cosine schedule
          UpperCAmelCase_							:      Optional[Any]						=   betas_for_alpha_bar(__snake_case	)
       else:
          raise NotImplementedError(f'''{beta_schedule} does is not implemented for {self.__class__}'''	)

       UpperCAmelCase_							:      Tuple						=   1.0 - self.betas
       UpperCAmelCase_							:      int						=   torch.cumprod(self.alphas       ,   dim=0	)

       #  set all values
       self.set_timesteps(__snake_case       ,   __snake_case       ,   __snake_case	)





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   List[Any]=None	):





       '''simple docstring'''
       if schedule_timesteps is None:
          UpperCAmelCase_							:      Any						=   self.timesteps

       UpperCAmelCase_							:      Optional[Any]						=   (schedule_timesteps == timestep).nonzero()

       # The sigma index that is taken for the **very** first `step`
       # is always the second index (or the last index if there is only 1)
       # This way we can ensure we don't accidentally skip a sigma in
       # case we start in the middle of the denoising schedule (e.g. for image-to-image)
       if len(self._index_counter	) == 0:
          UpperCAmelCase_							:      Optional[int]						=   1 if len(__snake_case	) > 1 else 0
       else:
          UpperCAmelCase_							:      Optional[int]						=   timestep.cpu().item() if torch.is_tensor(__snake_case	) else timestep
          UpperCAmelCase_							:      str						=   self._index_counter[timestep_int]

       return indices[pos].item()





    @property
    def 					_lowerCamelCase (	self				:   Dict	):





       '''simple docstring'''
       # standard deviation of the initial noise distribution
       if self.config.timestep_spacing in ["linspace", "trailing"]:
          return self.sigmas.max()

       return (self.sigmas.max() ** 2 + 1) ** 0.5





    def 					_lowerCamelCase (	self				:   Optional[Any]       ,   __snake_case				:   torch.FloatTensor       ,   __snake_case				:   Union[float, torch.FloatTensor]       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   self.index_for_timestep(__snake_case	)

       if self.state_in_first_order:
          UpperCAmelCase_							:      str						=   self.sigmas[step_index]
       else:
          UpperCAmelCase_							:      Optional[int]						=   self.sigmas_interpol[step_index]

       UpperCAmelCase_							:      Any						=   sample / ((sigma**2 + 1) ** 0.5)
       return sample





    def 					_lowerCamelCase (	self				:   Any       ,   __snake_case				:   int       ,   __snake_case				:   Union[str, torch.device] = None       ,   __snake_case				:   Optional[int] = None       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[int]						=   num_inference_steps

       UpperCAmelCase_							:      List[str]						=   num_train_timesteps or self.config.num_train_timesteps

       # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
       if self.config.timestep_spacing == "linspace":
          UpperCAmelCase_							:      Tuple						=   np.linspace(0       ,   num_train_timesteps - 1       ,   __snake_case       ,   dtype=__snake_case	)[::-1].copy()
       elif self.config.timestep_spacing == "leading":
          UpperCAmelCase_							:      Union[str, Any]						=   num_train_timesteps // self.num_inference_steps
          # creates integer timesteps by multiplying by ratio
          # casting to int to avoid issues when num_inference_step is power of 3
          UpperCAmelCase_							:      Tuple						=   (np.arange(0       ,   __snake_case	) * step_ratio).round()[::-1].copy().astype(__snake_case	)
          timesteps += self.config.steps_offset
       elif self.config.timestep_spacing == "trailing":
          UpperCAmelCase_							:      Optional[Any]						=   num_train_timesteps / self.num_inference_steps
          # creates integer timesteps by multiplying by ratio
          # casting to int to avoid issues when num_inference_step is power of 3
          UpperCAmelCase_							:      List[str]						=   (np.arange(__snake_case       ,   0       ,   -step_ratio	)).round().copy().astype(__snake_case	)
          timesteps -= 1
       else:
          raise ValueError(
              f'''{self.config.timestep_spacing} is not supported. Please make sure to choose one of \'linspace\', \'leading\' or \'trailing\'.'''	)

       UpperCAmelCase_							:      Any						=   np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5	)
       UpperCAmelCase_							:      Dict						=   torch.from_numpy(np.log(__snake_case	)	).to(__snake_case	)

       UpperCAmelCase_							:      Optional[Any]						=   np.interp(__snake_case       ,   np.arange(0       ,   len(__snake_case	)	)       ,   __snake_case	)
       UpperCAmelCase_							:      Optional[int]						=   np.concatenate([sigmas, [0.0]]	).astype(np.floataa	)
       UpperCAmelCase_							:      int						=   torch.from_numpy(__snake_case	).to(device=__snake_case	)

       # interpolate sigmas
       UpperCAmelCase_							:      Optional[int]						=   sigmas.log().lerp(sigmas.roll(1	).log()       ,   0.5	).exp()

       UpperCAmelCase_							:      str						=   torch.cat([sigmas[:1], sigmas[1:].repeat_interleave(2	), sigmas[-1:]]	)
       UpperCAmelCase_							:      Optional[int]						=   torch.cat(
           [sigmas_interpol[:1], sigmas_interpol[1:].repeat_interleave(2	), sigmas_interpol[-1:]]	)

       if str(__snake_case	).startswith('''mps'''	):
          # mps does not support float64
          UpperCAmelCase_							:      Union[str, Any]						=   torch.from_numpy(__snake_case	).to(__snake_case       ,   dtype=torch.floataa	)
       else:
          UpperCAmelCase_							:      str						=   torch.from_numpy(__snake_case	).to(__snake_case	)

       # interpolate timesteps
       UpperCAmelCase_							:      int						=   self.sigma_to_t(__snake_case	).to(__snake_case       ,   dtype=timesteps.dtype	)
       UpperCAmelCase_							:      Dict						=   torch.stack((timesteps_interpol[1:-1, None], timesteps[1:, None])       ,   dim=-1	).flatten()

       UpperCAmelCase_							:      Any						=   torch.cat([timesteps[:1], interleaved_timesteps]	)

       UpperCAmelCase_							:      List[Any]						=   None

       # for exp beta schedules, such as the one for `pipeline_shap_e.py`
       # we need an index counter
       UpperCAmelCase_							:      str						=   defaultdict(__snake_case	)





    def 					_lowerCamelCase (	self				:   int       ,   __snake_case				:   str	):





       '''simple docstring'''
       # get log sigma
       UpperCAmelCase_							:      Optional[Any]						=   sigma.log()

       # get distribution
       UpperCAmelCase_							:      List[Any]						=   log_sigma - self.log_sigmas[:, None]

       # get sigmas range
       UpperCAmelCase_							:      Optional[int]						=   dists.ge(0	).cumsum(dim=0	).argmax(dim=0	).clamp(max=self.log_sigmas.shape[0] - 2	)
       UpperCAmelCase_							:      Optional[int]						=   low_idx + 1

       UpperCAmelCase_							:      Any						=   self.log_sigmas[low_idx]
       UpperCAmelCase_							:      Union[str, Any]						=   self.log_sigmas[high_idx]

       # interpolate sigmas
       UpperCAmelCase_							:      Optional[int]						=   (low - log_sigma) / (low - high)
       UpperCAmelCase_							:      Union[str, Any]						=   w.clamp(0       ,   1	)

       # transform interpolation to time range
       UpperCAmelCase_							:      int						=   (1 - w) * low_idx + w * high_idx
       UpperCAmelCase_							:      Union[str, Any]						=   t.view(sigma.shape	)
       return t





    @property
    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       return self.sample is None





    def 					_lowerCamelCase (	self				:   Any       ,   __snake_case				:   Union[torch.FloatTensor, np.ndarray]       ,   __snake_case				:   Union[float, torch.FloatTensor]       ,   __snake_case				:   Union[torch.FloatTensor, np.ndarray]       ,   __snake_case				:   bool = True       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   self.index_for_timestep(__snake_case	)

       # advance index counter by 1
       UpperCAmelCase_							:      Union[str, Any]						=   timestep.cpu().item() if torch.is_tensor(__snake_case	) else timestep
       self._index_counter[timestep_int] += 1

       if self.state_in_first_order:
          UpperCAmelCase_							:      Optional[Any]						=   self.sigmas[step_index]
          UpperCAmelCase_							:      Optional[Any]						=   self.sigmas_interpol[step_index + 1]
          UpperCAmelCase_							:      int						=   self.sigmas[step_index + 1]
       else:
          # 2nd order / KDPM2's method
          UpperCAmelCase_							:      Dict						=   self.sigmas[step_index - 1]
          UpperCAmelCase_							:      Dict						=   self.sigmas_interpol[step_index]
          UpperCAmelCase_							:      Union[str, Any]						=   self.sigmas[step_index]

       # currently only gamma=0 is supported. This usually works best anyways.
       # We can support gamma in the future but then need to scale the timestep before
       # passing it to the model which requires a change in API
       UpperCAmelCase_							:      Optional[int]						=   0
       UpperCAmelCase_							:      Tuple						=   sigma * (gamma + 1)  # Note: sigma_hat == sigma for now

       # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise
       if self.config.prediction_type == "epsilon":
          UpperCAmelCase_							:      List[Any]						=   sigma_hat if self.state_in_first_order else sigma_interpol
          UpperCAmelCase_							:      Optional[int]						=   sample - sigma_input * model_output
       elif self.config.prediction_type == "v_prediction":
          UpperCAmelCase_							:      Any						=   sigma_hat if self.state_in_first_order else sigma_interpol
          UpperCAmelCase_							:      List[str]						=   model_output * (-sigma_input / (sigma_input**2 + 1) ** 0.5) + (
              sample / (sigma_input**2 + 1)
          )
       elif self.config.prediction_type == "sample":
          raise NotImplementedError('''prediction_type not implemented yet: sample'''	)
       else:
          raise ValueError(
              f'''prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`'''	)

       if self.state_in_first_order:
          # 2. Convert to an ODE derivative for 1st order
          UpperCAmelCase_							:      Optional[Any]						=   (sample - pred_original_sample) / sigma_hat
          # 3. delta timestep
          UpperCAmelCase_							:      List[Any]						=   sigma_interpol - sigma_hat

          # store for 2nd order step
          UpperCAmelCase_							:      List[Any]						=   sample
       else:
          # DPM-Solver-2
          # 2. Convert to an ODE derivative for 2nd order
          UpperCAmelCase_							:      str						=   (sample - pred_original_sample) / sigma_interpol

          # 3. delta timestep
          UpperCAmelCase_							:      List[str]						=   sigma_next - sigma_hat

          UpperCAmelCase_							:      Union[str, Any]						=   self.sample
          UpperCAmelCase_							:      List[str]						=   None

       UpperCAmelCase_							:      List[str]						=   sample + derivative * dt

       if not return_dict:
          return (prev_sample,)

       return SchedulerOutput(prev_sample=__snake_case	)





    def 					_lowerCamelCase (	self				:   Union[str, Any]       ,   __snake_case				:   torch.FloatTensor       ,   __snake_case				:   torch.FloatTensor       ,   __snake_case				:   torch.FloatTensor       ,   ):





       '''simple docstring'''
       # Make sure sigmas and timesteps have the same device and dtype as original_samples
       UpperCAmelCase_							:      str						=   self.sigmas.to(device=original_samples.device       ,   dtype=original_samples.dtype	)
       if original_samples.device.type == "mps" and torch.is_floating_point(__snake_case	):
          # mps does not support float64
          UpperCAmelCase_							:      Optional[int]						=   self.timesteps.to(original_samples.device       ,   dtype=torch.floataa	)
          UpperCAmelCase_							:      Tuple						=   timesteps.to(original_samples.device       ,   dtype=torch.floataa	)
       else:
          UpperCAmelCase_							:      Tuple						=   self.timesteps.to(original_samples.device	)
          UpperCAmelCase_							:      List[str]						=   timesteps.to(original_samples.device	)

       UpperCAmelCase_							:      Tuple						=   [self.index_for_timestep(__snake_case       ,   __snake_case	) for t in timesteps]

       UpperCAmelCase_							:      int						=   sigmas[step_indices].flatten()
       while len(sigma.shape	) < len(original_samples.shape	):
          UpperCAmelCase_							:      Any						=   sigma.unsqueeze(-1	)

       UpperCAmelCase_							:      Dict						=   original_samples + noise * sigma
       return noisy_samples





    def __len__(	self				:   Dict	):





       '''simple docstring'''
       return self.config.num_train_timesteps



import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           XLMForMultipleChoice,
           XLMForQuestionAnswering,
           XLMForQuestionAnsweringSimple,
           XLMForSequenceClassification,
           XLMForTokenClassification,
           XLMModel,
           XLMWithLMHeadModel,
       )
       from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST





class  lowerCAmelCase__:






    '''simple docstring'''





    def __init__(	self				:   int       ,   __snake_case				:   List[Any]       ,   __snake_case				:   List[Any]=13       ,   __snake_case				:   str=7       ,   __snake_case				:   Union[str, Any]=True       ,   __snake_case				:   List[Any]=True       ,   __snake_case				:   str=True       ,   __snake_case				:   Optional[int]=True       ,   __snake_case				:   Optional[int]=True       ,   __snake_case				:   List[str]=False       ,   __snake_case				:   List[str]=False       ,   __snake_case				:   Tuple=False       ,   __snake_case				:   List[str]=2       ,   __snake_case				:   Optional[int]=99       ,   __snake_case				:   Tuple=0       ,   __snake_case				:   int=32       ,   __snake_case				:   Optional[int]=5       ,   __snake_case				:   str=4       ,   __snake_case				:   str=0.1       ,   __snake_case				:   Optional[int]=0.1       ,   __snake_case				:   List[str]=512       ,   __snake_case				:   Tuple=2       ,   __snake_case				:   List[Any]=0.02       ,   __snake_case				:   Any=2       ,   __snake_case				:   Optional[int]=4       ,   __snake_case				:   Optional[Any]="last"       ,   __snake_case				:   Dict=True       ,   __snake_case				:   Any=None       ,   __snake_case				:   str=0       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   parent
       UpperCAmelCase_							:      Optional[Any]						=   batch_size
       UpperCAmelCase_							:      Union[str, Any]						=   seq_length
       UpperCAmelCase_							:      List[Any]						=   is_training
       UpperCAmelCase_							:      List[Any]						=   use_input_lengths
       UpperCAmelCase_							:      Dict						=   use_token_type_ids
       UpperCAmelCase_							:      Union[str, Any]						=   use_labels
       UpperCAmelCase_							:      int						=   gelu_activation
       UpperCAmelCase_							:      str						=   sinusoidal_embeddings
       UpperCAmelCase_							:      List[str]						=   causal
       UpperCAmelCase_							:      Tuple						=   asm
       UpperCAmelCase_							:      List[Any]						=   n_langs
       UpperCAmelCase_							:      Union[str, Any]						=   vocab_size
       UpperCAmelCase_							:      Any						=   n_special
       UpperCAmelCase_							:      Dict						=   hidden_size
       UpperCAmelCase_							:      Dict						=   num_hidden_layers
       UpperCAmelCase_							:      str						=   num_attention_heads
       UpperCAmelCase_							:      List[str]						=   hidden_dropout_prob
       UpperCAmelCase_							:      Any						=   attention_probs_dropout_prob
       UpperCAmelCase_							:      Tuple						=   max_position_embeddings
       UpperCAmelCase_							:      Union[str, Any]						=   type_sequence_label_size
       UpperCAmelCase_							:      Union[str, Any]						=   initializer_range
       UpperCAmelCase_							:      Tuple						=   num_labels
       UpperCAmelCase_							:      List[Any]						=   num_choices
       UpperCAmelCase_							:      Any						=   summary_type
       UpperCAmelCase_							:      Optional[int]						=   use_proj
       UpperCAmelCase_							:      List[str]						=   scope
       UpperCAmelCase_							:      List[str]						=   bos_token_id





    def 					_lowerCamelCase (	self				:   Optional[int]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Tuple						=   ids_tensor([self.batch_size, self.seq_length]       ,   self.vocab_size	)
       UpperCAmelCase_							:      Union[str, Any]						=   random_attention_mask([self.batch_size, self.seq_length]	)

       UpperCAmelCase_							:      Tuple						=   None
       if self.use_input_lengths:
          UpperCAmelCase_							:      List[str]						=   (
              ids_tensor([self.batch_size]       ,   vocab_size=2	) + self.seq_length - 2
          )  # small variation of seq_length

       UpperCAmelCase_							:      int						=   None
       if self.use_token_type_ids:
          UpperCAmelCase_							:      List[str]						=   ids_tensor([self.batch_size, self.seq_length]       ,   self.n_langs	)

       UpperCAmelCase_							:      Optional[int]						=   None
       UpperCAmelCase_							:      Union[str, Any]						=   None
       UpperCAmelCase_							:      str						=   None
       if self.use_labels:
          UpperCAmelCase_							:      Any						=   ids_tensor([self.batch_size]       ,   self.type_sequence_label_size	)
          UpperCAmelCase_							:      List[str]						=   ids_tensor([self.batch_size, self.seq_length]       ,   self.num_labels	)
          UpperCAmelCase_							:      List[Any]						=   ids_tensor([self.batch_size]       ,   2	).float()
          UpperCAmelCase_							:      Optional[Any]						=   ids_tensor([self.batch_size]       ,   self.num_choices	)

       UpperCAmelCase_							:      List[Any]						=   self.get_config()

       return (
           config,
           input_ids,
           token_type_ids,
           input_lengths,
           sequence_labels,
           token_labels,
           is_impossible_labels,
           choice_labels,
           input_mask,
       )





    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       return XLMConfig(
           vocab_size=self.vocab_size       ,   n_special=self.n_special       ,   emb_dim=self.hidden_size       ,   n_layers=self.num_hidden_layers       ,   n_heads=self.num_attention_heads       ,   dropout=self.hidden_dropout_prob       ,   attention_dropout=self.attention_probs_dropout_prob       ,   gelu_activation=self.gelu_activation       ,   sinusoidal_embeddings=self.sinusoidal_embeddings       ,   asm=self.asm       ,   causal=self.causal       ,   n_langs=self.n_langs       ,   max_position_embeddings=self.max_position_embeddings       ,   initializer_range=self.initializer_range       ,   summary_type=self.summary_type       ,   use_proj=self.use_proj       ,   num_labels=self.num_labels       ,   bos_token_id=self.bos_token_id       ,   )





    def 					_lowerCamelCase (	self				:   Optional[Any]       ,   __snake_case				:   int       ,   __snake_case				:   int       ,   __snake_case				:   List[Any]       ,   __snake_case				:   List[Any]       ,   __snake_case				:   List[str]       ,   __snake_case				:   List[str]       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Dict       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   XLMModel(config=__snake_case	)
       model.to(__snake_case	)
       model.eval()
       UpperCAmelCase_							:      Optional[int]						=   model(__snake_case       ,   lengths=__snake_case       ,   langs=__snake_case	)
       UpperCAmelCase_							:      Any						=   model(__snake_case       ,   langs=__snake_case	)
       UpperCAmelCase_							:      Any						=   model(__snake_case	)
       self.parent.assertEqual(result.last_hidden_state.shape       ,   (self.batch_size, self.seq_length, self.hidden_size)	)





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   Dict       ,   __snake_case				:   Any       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Dict       ,   __snake_case				:   str       ,   __snake_case				:   Tuple       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   int       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   XLMWithLMHeadModel(__snake_case	)
       model.to(__snake_case	)
       model.eval()

       UpperCAmelCase_							:      Optional[Any]						=   model(__snake_case       ,   token_type_ids=__snake_case       ,   labels=__snake_case	)
       self.parent.assertEqual(result.loss.shape       ,   ()	)
       self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.seq_length, self.vocab_size)	)





    def 					_lowerCamelCase (	self				:   Optional[int]       ,   __snake_case				:   Tuple       ,   __snake_case				:   Any       ,   __snake_case				:   Tuple       ,   __snake_case				:   List[str]       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   List[str]       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   str       ,   __snake_case				:   List[str]       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   XLMForQuestionAnsweringSimple(__snake_case	)
       model.to(__snake_case	)
       model.eval()

       UpperCAmelCase_							:      Optional[int]						=   model(__snake_case	)

       UpperCAmelCase_							:      Any						=   model(__snake_case       ,   start_positions=__snake_case       ,   end_positions=__snake_case	)
       UpperCAmelCase_							:      Optional[Any]						=   outputs
       self.parent.assertEqual(result.start_logits.shape       ,   (self.batch_size, self.seq_length)	)
       self.parent.assertEqual(result.end_logits.shape       ,   (self.batch_size, self.seq_length)	)





    def 					_lowerCamelCase (	self				:   Any       ,   __snake_case				:   int       ,   __snake_case				:   Dict       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   List[str]       ,   __snake_case				:   Tuple       ,   __snake_case				:   str       ,   __snake_case				:   Tuple       ,   __snake_case				:   int       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      List[str]						=   XLMForQuestionAnswering(__snake_case	)
       model.to(__snake_case	)
       model.eval()

       UpperCAmelCase_							:      Union[str, Any]						=   model(__snake_case	)

       UpperCAmelCase_							:      List[str]						=   model(
           __snake_case       ,   start_positions=__snake_case       ,   end_positions=__snake_case       ,   cls_index=__snake_case       ,   is_impossible=__snake_case       ,   p_mask=__snake_case       ,   )

       UpperCAmelCase_							:      Optional[Any]						=   model(
           __snake_case       ,   start_positions=__snake_case       ,   end_positions=__snake_case       ,   cls_index=__snake_case       ,   is_impossible=__snake_case       ,   )

       ((UpperCAmelCase_)		,      )							:      Union[str, Any]						=   result_with_labels.to_tuple()

       UpperCAmelCase_							:      Optional[int]						=   model(__snake_case       ,   start_positions=__snake_case       ,   end_positions=__snake_case	)

       ((UpperCAmelCase_)		,      )							:      str						=   result_with_labels.to_tuple()

       self.parent.assertEqual(result_with_labels.loss.shape       ,   ()	)
       self.parent.assertEqual(result.start_top_log_probs.shape       ,   (self.batch_size, model.config.start_n_top)	)
       self.parent.assertEqual(result.start_top_index.shape       ,   (self.batch_size, model.config.start_n_top)	)
       self.parent.assertEqual(
           result.end_top_log_probs.shape       ,   (self.batch_size, model.config.start_n_top * model.config.end_n_top)	)
       self.parent.assertEqual(
           result.end_top_index.shape       ,   (self.batch_size, model.config.start_n_top * model.config.end_n_top)	)
       self.parent.assertEqual(result.cls_logits.shape       ,   (self.batch_size,)	)





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   Dict       ,   __snake_case				:   str       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   Dict       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Dict       ,   __snake_case				:   List[str]       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   Any       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      List[str]						=   XLMForSequenceClassification(__snake_case	)
       model.to(__snake_case	)
       model.eval()

       UpperCAmelCase_							:      Union[str, Any]						=   model(__snake_case	)
       UpperCAmelCase_							:      Optional[int]						=   model(__snake_case       ,   labels=__snake_case	)
       self.parent.assertEqual(result.loss.shape       ,   ()	)
       self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.type_sequence_label_size)	)





    def 					_lowerCamelCase (	self				:   Any       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Any       ,   __snake_case				:   List[str]       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   Tuple       ,   __snake_case				:   str       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Tuple       ,   __snake_case				:   int       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   self.num_labels
       UpperCAmelCase_							:      Optional[int]						=   XLMForTokenClassification(__snake_case	)
       model.to(__snake_case	)
       model.eval()

       UpperCAmelCase_							:      List[str]						=   model(__snake_case       ,   attention_mask=__snake_case       ,   labels=__snake_case	)
       self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.seq_length, self.num_labels)	)





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   str       ,   __snake_case				:   List[str]       ,   __snake_case				:   Any       ,   __snake_case				:   Tuple       ,   __snake_case				:   Any       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Dict       ,   __snake_case				:   Dict       ,   __snake_case				:   Optional[int]       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   self.num_choices
       UpperCAmelCase_							:      int						=   XLMForMultipleChoice(config=__snake_case	)
       model.to(__snake_case	)
       model.eval()
       UpperCAmelCase_							:      int						=   input_ids.unsqueeze(1	).expand(-1       ,   self.num_choices       ,   -1	).contiguous()
       UpperCAmelCase_							:      int						=   token_type_ids.unsqueeze(1	).expand(-1       ,   self.num_choices       ,   -1	).contiguous()
       UpperCAmelCase_							:      List[Any]						=   input_mask.unsqueeze(1	).expand(-1       ,   self.num_choices       ,   -1	).contiguous()
       UpperCAmelCase_							:      Any						=   model(
           __snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case       ,   labels=__snake_case       ,   )
       self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.num_choices)	)





    def 					_lowerCamelCase (	self				:   List[str]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   self.prepare_config_and_inputs()
       (
           (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      
       )							:      Union[str, Any]						=   config_and_inputs
       UpperCAmelCase_							:      str						=   {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''lengths''': input_lengths}
       return config, inputs_dict





@require_torch
class  lowerCAmelCase__(    snake_case__    ,    snake_case__    ,    snake_case__    ,    unittest.TestCase						):






    '''simple docstring'''





    A_  :     Dict			= (
        (
            XLMModel,
            XLMWithLMHeadModel,
            XLMForQuestionAnswering,
            XLMForSequenceClassification,
            XLMForQuestionAnsweringSimple,
            XLMForTokenClassification,
            XLMForMultipleChoice,
        )
        if is_torch_available()
        else ()
    )
    A_  :     List[str]			= (
        (XLMWithLMHeadModel,) if is_torch_available() else ()
    )  # TODO (PVP): Check other models whether language generation is also applicable
    A_  :     Optional[int]			= (
        {
            'feature-extraction': XLMModel,
            'fill-mask': XLMWithLMHeadModel,
            'question-answering': XLMForQuestionAnsweringSimple,
            'text-classification': XLMForSequenceClassification,
            'text-generation': XLMWithLMHeadModel,
            'token-classification': XLMForTokenClassification,
            'zero-shot': XLMForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Optional[Any]	):





       '''simple docstring'''
       if (
           pipeline_test_casse_name == "QAPipelineTests"
           and tokenizer_name is not None
           and not tokenizer_name.endswith('''Fast'''	)
       ):
          # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
          # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
          # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
          return True

       return False





    def 					_lowerCamelCase (	self				:   Union[str, Any]       ,   __snake_case				:   Any       ,   __snake_case				:   List[Any]       ,   __snake_case				:   str=False	):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   super()._prepare_for_class(__snake_case       ,   __snake_case       ,   return_labels=__snake_case	)

       if return_labels:
          if model_class.__name__ == "XLMForQuestionAnswering":
             UpperCAmelCase_							:      List[str]						=   torch.zeros(
                 self.model_tester.batch_size       ,   dtype=torch.long       ,   device=__snake_case	)
             UpperCAmelCase_							:      Dict						=   torch.zeros(
                 self.model_tester.batch_size       ,   dtype=torch.long       ,   device=__snake_case	)

       return inputs_dict





    def 					_lowerCamelCase (	self				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   XLMModelTester(self	)
       UpperCAmelCase_							:      List[Any]						=   ConfigTester(self       ,   config_class=__snake_case       ,   emb_dim=37	)





    def 					_lowerCamelCase (	self				:   Union[str, Any]	):





       '''simple docstring'''
       self.config_tester.run_common_tests()





    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_model(*__snake_case	)





    def 					_lowerCamelCase (	self				:   Union[str, Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Tuple						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_lm_head(*__snake_case	)





    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_simple_qa(*__snake_case	)





    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_qa(*__snake_case	)





    def 					_lowerCamelCase (	self				:   List[str]	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_sequence_classif(*__snake_case	)





    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_token_classif(*__snake_case	)





    def 					_lowerCamelCase (	self				:   str	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[int]						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_for_multiple_choice(*__snake_case	)





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   str       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   List[Any]       ,   __snake_case				:   str       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Union[str, Any]=False       ,   __snake_case				:   Optional[Any]=1	):





       '''simple docstring'''
       self.assertIsInstance(__snake_case       ,   __snake_case	)
       self.assertListEqual(
           [isinstance(__snake_case       ,   __snake_case	) for iter_attentions in attentions]       ,   [True] * len(__snake_case	)	)
       self.assertEqual(len(__snake_case	)       ,   (max_length - min_length) * num_beam_groups	)

       for idx, iter_attentions in enumerate(__snake_case	):
          # adds PAD dummy token
          UpperCAmelCase_							:      Dict						=   min_length + idx + 1
          UpperCAmelCase_							:      List[Any]						=   min_length + idx + 1

          UpperCAmelCase_							:      Optional[int]						=   (
              batch_size * num_beam_groups,
              config.num_attention_heads,
              tgt_len,
              src_len,
          )
          # check attn size
          self.assertListEqual(
              [layer_attention.shape for layer_attention in iter_attentions]       ,   [expected_shape] * len(__snake_case	)	)





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   List[str]       ,   __snake_case				:   int       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Dict       ,   __snake_case				:   Any       ,   __snake_case				:   Optional[Any]=False       ,   __snake_case				:   str=1	):





       '''simple docstring'''
       self.assertIsInstance(__snake_case       ,   __snake_case	)
       self.assertListEqual(
           [isinstance(__snake_case       ,   __snake_case	) for iter_hidden_states in hidden_states]       ,   [True] * len(__snake_case	)       ,   )
       self.assertEqual(len(__snake_case	)       ,   (max_length - min_length) * num_beam_groups	)

       for idx, iter_hidden_states in enumerate(__snake_case	):
          # adds PAD dummy token
          UpperCAmelCase_							:      str						=   min_length + idx + 1
          UpperCAmelCase_							:      int						=   (batch_size * num_beam_groups, seq_len, config.hidden_size)
          # check hidden size
          self.assertListEqual(
              [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]       ,   [expected_shape] * len(__snake_case	)       ,   )
       pass





    @slow
    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          UpperCAmelCase_							:      Any						=   XLMModel.from_pretrained(__snake_case	)
          self.assertIsNotNone(__snake_case	)





@require_torch
class  lowerCAmelCase__(    unittest.TestCase						):






    '''simple docstring'''





    @slow
    def 					_lowerCamelCase (	self				:   str	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[int]						=   XLMWithLMHeadModel.from_pretrained('''xlm-mlm-en-2048'''	)
       model.to(__snake_case	)
       UpperCAmelCase_							:      str						=   torch.tensor([[14, 447]]       ,   dtype=torch.long       ,   device=__snake_case	)  # the president
       UpperCAmelCase_							:      Union[str, Any]						=   [
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
       ]  # the president the president the president the president the president the president the president the president the president the president
       # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
       UpperCAmelCase_							:      Dict						=   model.generate(__snake_case       ,   do_sample=__snake_case	)
       self.assertListEqual(output_ids[0].cpu().numpy().tolist()       ,   __snake_case	)