


from typing import Callable, List, Optional, Tuple, Union

import torch
from transformers import CLIPTextModel, CLIPTokenizer

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin, TransformeraDModel, VQModel
from ...schedulers import VQDiffusionScheduler
from ...utils import logging
from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput


__UpperCamelCase				:  Optional[Any]      					=							logging.get_logger(__name__)  # pylint: disable=invalid-name





class  lowerCAmelCase__(    snake_case__    ,    snake_case__						):






    '''simple docstring'''





    @register_to_config
    def __init__(	self				:   int       ,   __snake_case				:   bool       ,   __snake_case				:   Optional[int] = None       ,   __snake_case				:   Optional[int] = None	):





       '''simple docstring'''
       super().__init__()

       UpperCAmelCase_							:      int						=   learnable

       if self.learnable:
          assert hidden_size is not None, "learnable=True requires `hidden_size` to be set"
          assert length is not None, "learnable=True requires `length` to be set"

          UpperCAmelCase_							:      str						=   torch.zeros(__snake_case       ,   __snake_case	)
       else:
          UpperCAmelCase_							:      Any						=   None

       UpperCAmelCase_							:      Dict						=   torch.nn.Parameter(__snake_case	)





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     VQModel
    A_  :     CLIPTextModel
    A_  :     CLIPTokenizer
    A_  :     TransformeraDModel
    A_  :     LearnedClassifierFreeSamplingEmbeddings
    A_  :     VQDiffusionScheduler





    def __init__(	self				:   str       ,   __snake_case				:   VQModel       ,   __snake_case				:   CLIPTextModel       ,   __snake_case				:   CLIPTokenizer       ,   __snake_case				:   TransformeraDModel       ,   __snake_case				:   VQDiffusionScheduler       ,   __snake_case				:   LearnedClassifierFreeSamplingEmbeddings       ,   ):





       '''simple docstring'''
       super().__init__()

       self.register_modules(
           vqvae=__snake_case       ,   transformer=__snake_case       ,   text_encoder=__snake_case       ,   tokenizer=__snake_case       ,   scheduler=__snake_case       ,   learned_classifier_free_sampling_embeddings=__snake_case       ,   )





    def 					_lowerCamelCase (	self				:   Tuple       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   List[str]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Dict						=   len(__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else 1

       # get prompt text embeddings
       UpperCAmelCase_							:      Union[str, Any]						=   self.tokenizer(
           __snake_case       ,   padding='''max_length'''       ,   max_length=self.tokenizer.model_max_length       ,   return_tensors='''pt'''       ,   )
       UpperCAmelCase_							:      Any						=   text_inputs.input_ids

       if text_input_ids.shape[-1] > self.tokenizer.model_max_length:
          UpperCAmelCase_							:      Any						=   self.tokenizer.batch_decode(text_input_ids[:, self.tokenizer.model_max_length :]	)
          logger.warning(
              '''The following part of your input was truncated because CLIP can only handle sequences up to'''
              f''' {self.tokenizer.model_max_length} tokens: {removed_text}'''	)
          UpperCAmelCase_							:      Dict						=   text_input_ids[:, : self.tokenizer.model_max_length]
       UpperCAmelCase_							:      Optional[int]						=   self.text_encoder(text_input_ids.to(self.device	)	)[0]

       # NOTE: This additional step of normalizing the text embeddings is from VQ-Diffusion.
       # While CLIP does normalize the pooled output of the text transformer when combining
       # the image and text embeddings, CLIP does not directly normalize the last hidden state.
       #
       # CLIP normalizing the pooled output.
       # https://github.com/huggingface/transformers/blob/d92e22d1f28324f513f3080e5c47c071a3916721/src/transformers/models/clip/modeling_clip.py#L1052-L1053
       UpperCAmelCase_							:      Optional[Any]						=   prompt_embeds / prompt_embeds.norm(dim=-1       ,   keepdim=__snake_case	)

       # duplicate text embeddings for each generation per prompt
       UpperCAmelCase_							:      List[Any]						=   prompt_embeds.repeat_interleave(__snake_case       ,   dim=0	)

       if do_classifier_free_guidance:
          if self.learned_classifier_free_sampling_embeddings.learnable:
             UpperCAmelCase_							:      List[str]						=   self.learned_classifier_free_sampling_embeddings.embeddings
             UpperCAmelCase_							:      Optional[int]						=   negative_prompt_embeds.unsqueeze(0	).repeat(__snake_case       ,   1       ,   1	)
          else:
             UpperCAmelCase_							:      int						=   [''''''] * batch_size

             UpperCAmelCase_							:      Dict						=   text_input_ids.shape[-1]
             UpperCAmelCase_							:      Dict						=   self.tokenizer(
                 __snake_case       ,   padding='''max_length'''       ,   max_length=__snake_case       ,   truncation=__snake_case       ,   return_tensors='''pt'''       ,   )
             UpperCAmelCase_							:      int						=   self.text_encoder(uncond_input.input_ids.to(self.device	)	)[0]
             # See comment for normalizing text embeddings
             UpperCAmelCase_							:      Any						=   negative_prompt_embeds / negative_prompt_embeds.norm(dim=-1       ,   keepdim=__snake_case	)

          # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
          UpperCAmelCase_							:      Dict						=   negative_prompt_embeds.shape[1]
          UpperCAmelCase_							:      Union[str, Any]						=   negative_prompt_embeds.repeat(1       ,   __snake_case       ,   1	)
          UpperCAmelCase_							:      Optional[int]						=   negative_prompt_embeds.view(batch_size * num_images_per_prompt       ,   __snake_case       ,   -1	)

          # For classifier free guidance, we need to do two forward passes.
          # Here we concatenate the unconditional and text embeddings into a single batch
          # to avoid doing two forward passes
          UpperCAmelCase_							:      str						=   torch.cat([negative_prompt_embeds, prompt_embeds]	)

       return prompt_embeds





    @torch.no_grad()
    def __call__(	self				:   Optional[Any]       ,   __snake_case				:   Union[str, List[str]]       ,   __snake_case				:   int = 100       ,   __snake_case				:   float = 5.0       ,   __snake_case				:   float = 1.0       ,   __snake_case				:   int = 1       ,   __snake_case				:   Optional[Union[torch.Generator, List[torch.Generator]]] = None       ,   __snake_case				:   Optional[torch.FloatTensor] = None       ,   __snake_case				:   Optional[str] = "pil"       ,   __snake_case				:   bool = True       ,   __snake_case				:   Optional[Callable[[int, int, torch.FloatTensor], None]] = None       ,   __snake_case				:   int = 1       ,   ):





       '''simple docstring'''
       if isinstance(__snake_case       ,   __snake_case	):
          UpperCAmelCase_							:      Union[str, Any]						=   1
       elif isinstance(__snake_case       ,   __snake_case	):
          UpperCAmelCase_							:      Optional[int]						=   len(__snake_case	)
       else:
          raise ValueError(f'''`prompt` has to be of type `str` or `list` but is {type(__snake_case	)}'''	)

       UpperCAmelCase_							:      Dict						=   batch_size * num_images_per_prompt

       UpperCAmelCase_							:      str						=   guidance_scale > 1.0

       UpperCAmelCase_							:      str						=   self._encode_prompt(__snake_case       ,   __snake_case       ,   __snake_case	)

       if (callback_steps is None) or (
           callback_steps is not None and (not isinstance(__snake_case       ,   __snake_case	) or callback_steps <= 0)
       ):
          raise ValueError(
              f'''`callback_steps` has to be a positive integer but is {callback_steps} of type'''
              f''' {type(__snake_case	)}.'''	)

       # get the initial completely masked latents unless the user supplied it

       UpperCAmelCase_							:      Tuple						=   (batch_size, self.transformer.num_latent_pixels)
       if latents is None:
          UpperCAmelCase_							:      List[str]						=   self.transformer.num_vector_embeds - 1
          UpperCAmelCase_							:      Tuple						=   torch.full(__snake_case       ,   __snake_case	).to(self.device	)
       else:
          if latents.shape != latents_shape:
             raise ValueError(f'''Unexpected latents shape, got {latents.shape}, expected {latents_shape}'''	)
          if (latents < 0).any() or (latents >= self.transformer.num_vector_embeds).any():
             raise ValueError(
                 '''Unexpected latents value(s). All latents be valid embedding indices i.e. in the range 0,'''
                 f''' {self.transformer.num_vector_embeds - 1} (inclusive).'''	)
          UpperCAmelCase_							:      Optional[int]						=   latents.to(self.device	)

       # set timesteps
       self.scheduler.set_timesteps(__snake_case       ,   device=self.device	)

       UpperCAmelCase_							:      str						=   self.scheduler.timesteps.to(self.device	)

       UpperCAmelCase_							:      Tuple						=   latents

       for i, t in enumerate(self.progress_bar(__snake_case	)	):
          # expand the sample if we are doing classifier free guidance
          UpperCAmelCase_							:      List[str]						=   torch.cat([sample] * 2	) if do_classifier_free_guidance else sample

          # predict the un-noised image
          # model_output == `log_p_x_0`
          UpperCAmelCase_							:      Optional[int]						=   self.transformer(__snake_case       ,   encoder_hidden_states=__snake_case       ,   timestep=__snake_case	).sample

          if do_classifier_free_guidance:
             UpperCAmelCase_		,      UpperCAmelCase_							:      Tuple						=   model_output.chunk(2	)
             UpperCAmelCase_							:      Optional[int]						=   model_output_uncond + guidance_scale * (model_output_text - model_output_uncond)
             model_output -= torch.logsumexp(__snake_case       ,   dim=1       ,   keepdim=__snake_case	)

          UpperCAmelCase_							:      Optional[int]						=   self.truncate(__snake_case       ,   __snake_case	)

          # remove `log(0)`'s (`-inf`s)
          UpperCAmelCase_							:      List[Any]						=   model_output.clamp(-70	)

          # compute the previous noisy sample x_t -> x_t-1
          UpperCAmelCase_							:      Dict						=   self.scheduler.step(__snake_case       ,   timestep=__snake_case       ,   sample=__snake_case       ,   generator=__snake_case	).prev_sample

          # call the callback, if provided
          if callback is not None and i % callback_steps == 0:
             callback(__snake_case       ,   __snake_case       ,   __snake_case	)

       UpperCAmelCase_							:      Dict						=   self.vqvae.config.vq_embed_dim
       UpperCAmelCase_							:      Any						=   (batch_size, self.transformer.height, self.transformer.width, embedding_channels)
       UpperCAmelCase_							:      str						=   self.vqvae.quantize.get_codebook_entry(__snake_case       ,   shape=__snake_case	)
       UpperCAmelCase_							:      Union[str, Any]						=   self.vqvae.decode(__snake_case       ,   force_not_quantize=__snake_case	).sample

       UpperCAmelCase_							:      Union[str, Any]						=   (image / 2 + 0.5).clamp(0       ,   1	)
       UpperCAmelCase_							:      List[str]						=   image.cpu().permute(0       ,   2       ,   3       ,   1	).numpy()

       if output_type == "pil":
          UpperCAmelCase_							:      Optional[int]						=   self.numpy_to_pil(__snake_case	)

       if not return_dict:
          return (image,)

       return ImagePipelineOutput(images=__snake_case	)





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   torch.FloatTensor       ,   __snake_case				:   float	):





       '''simple docstring'''
       UpperCAmelCase_		,      UpperCAmelCase_							:      Optional[int]						=   torch.sort(__snake_case       ,   1       ,   descending=__snake_case	)
       UpperCAmelCase_							:      Optional[int]						=   torch.exp(__snake_case	)
       UpperCAmelCase_							:      Tuple						=   sorted_p_x_0.cumsum(dim=1	) < truncation_rate

       # Ensure that at least the largest probability is not zeroed out
       UpperCAmelCase_							:      Any						=   torch.full_like(keep_mask[:, 0:1, :]       ,   __snake_case	)
       UpperCAmelCase_							:      Union[str, Any]						=   torch.cat((all_true, keep_mask)       ,   dim=1	)
       UpperCAmelCase_							:      str						=   keep_mask[:, :-1, :]

       UpperCAmelCase_							:      List[Any]						=   keep_mask.gather(1       ,   indices.argsort(1	)	)

       UpperCAmelCase_							:      Optional[Any]						=   log_p_x_0.clone()

       UpperCAmelCase_							:      Any						=   -torch.inf  # -inf = log(0)

       return rv



import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           XLMForMultipleChoice,
           XLMForQuestionAnswering,
           XLMForQuestionAnsweringSimple,
           XLMForSequenceClassification,
           XLMForTokenClassification,
           XLMModel,
           XLMWithLMHeadModel,
       )
       from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST





class  lowerCAmelCase__:






    '''simple docstring'''





    def __init__(	self				:   int       ,   __snake_case				:   List[Any]       ,   __snake_case				:   List[Any]=13       ,   __snake_case				:   str=7       ,   __snake_case				:   Union[str, Any]=True       ,   __snake_case				:   List[Any]=True       ,   __snake_case				:   str=True       ,   __snake_case				:   Optional[int]=True       ,   __snake_case				:   Optional[int]=True       ,   __snake_case				:   List[str]=False       ,   __snake_case				:   List[str]=False       ,   __snake_case				:   Tuple=False       ,   __snake_case				:   List[str]=2       ,   __snake_case				:   Optional[int]=99       ,   __snake_case				:   Tuple=0       ,   __snake_case				:   int=32       ,   __snake_case				:   Optional[int]=5       ,   __snake_case				:   str=4       ,   __snake_case				:   str=0.1       ,   __snake_case				:   Optional[int]=0.1       ,   __snake_case				:   List[str]=512       ,   __snake_case				:   Tuple=2       ,   __snake_case				:   List[Any]=0.02       ,   __snake_case				:   Any=2       ,   __snake_case				:   Optional[int]=4       ,   __snake_case				:   Optional[Any]="last"       ,   __snake_case				:   Dict=True       ,   __snake_case				:   Any=None       ,   __snake_case				:   str=0       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   parent
       UpperCAmelCase_							:      Optional[Any]						=   batch_size
       UpperCAmelCase_							:      Union[str, Any]						=   seq_length
       UpperCAmelCase_							:      List[Any]						=   is_training
       UpperCAmelCase_							:      List[Any]						=   use_input_lengths
       UpperCAmelCase_							:      Dict						=   use_token_type_ids
       UpperCAmelCase_							:      Union[str, Any]						=   use_labels
       UpperCAmelCase_							:      int						=   gelu_activation
       UpperCAmelCase_							:      str						=   sinusoidal_embeddings
       UpperCAmelCase_							:      List[str]						=   causal
       UpperCAmelCase_							:      Tuple						=   asm
       UpperCAmelCase_							:      List[Any]						=   n_langs
       UpperCAmelCase_							:      Union[str, Any]						=   vocab_size
       UpperCAmelCase_							:      Any						=   n_special
       UpperCAmelCase_							:      Dict						=   hidden_size
       UpperCAmelCase_							:      Dict						=   num_hidden_layers
       UpperCAmelCase_							:      str						=   num_attention_heads
       UpperCAmelCase_							:      List[str]						=   hidden_dropout_prob
       UpperCAmelCase_							:      Any						=   attention_probs_dropout_prob
       UpperCAmelCase_							:      Tuple						=   max_position_embeddings
       UpperCAmelCase_							:      Union[str, Any]						=   type_sequence_label_size
       UpperCAmelCase_							:      Union[str, Any]						=   initializer_range
       UpperCAmelCase_							:      Tuple						=   num_labels
       UpperCAmelCase_							:      List[Any]						=   num_choices
       UpperCAmelCase_							:      Any						=   summary_type
       UpperCAmelCase_							:      Optional[int]						=   use_proj
       UpperCAmelCase_							:      List[str]						=   scope
       UpperCAmelCase_							:      List[str]						=   bos_token_id





    def 					_lowerCamelCase (	self				:   Optional[int]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Tuple						=   ids_tensor([self.batch_size, self.seq_length]       ,   self.vocab_size	)
       UpperCAmelCase_							:      Union[str, Any]						=   random_attention_mask([self.batch_size, self.seq_length]	)

       UpperCAmelCase_							:      Tuple						=   None
       if self.use_input_lengths:
          UpperCAmelCase_							:      List[str]						=   (
              ids_tensor([self.batch_size]       ,   vocab_size=2	) + self.seq_length - 2
          )  # small variation of seq_length

       UpperCAmelCase_							:      int						=   None
       if self.use_token_type_ids:
          UpperCAmelCase_							:      List[str]						=   ids_tensor([self.batch_size, self.seq_length]       ,   self.n_langs	)

       UpperCAmelCase_							:      Optional[int]						=   None
       UpperCAmelCase_							:      Union[str, Any]						=   None
       UpperCAmelCase_							:      str						=   None
       if self.use_labels:
          UpperCAmelCase_							:      Any						=   ids_tensor([self.batch_size]       ,   self.type_sequence_label_size	)
          UpperCAmelCase_							:      List[str]						=   ids_tensor([self.batch_size, self.seq_length]       ,   self.num_labels	)
          UpperCAmelCase_							:      List[Any]						=   ids_tensor([self.batch_size]       ,   2	).float()
          UpperCAmelCase_							:      Optional[Any]						=   ids_tensor([self.batch_size]       ,   self.num_choices	)

       UpperCAmelCase_							:      List[Any]						=   self.get_config()

       return (
           config,
           input_ids,
           token_type_ids,
           input_lengths,
           sequence_labels,
           token_labels,
           is_impossible_labels,
           choice_labels,
           input_mask,
       )





    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       return XLMConfig(
           vocab_size=self.vocab_size       ,   n_special=self.n_special       ,   emb_dim=self.hidden_size       ,   n_layers=self.num_hidden_layers       ,   n_heads=self.num_attention_heads       ,   dropout=self.hidden_dropout_prob       ,   attention_dropout=self.attention_probs_dropout_prob       ,   gelu_activation=self.gelu_activation       ,   sinusoidal_embeddings=self.sinusoidal_embeddings       ,   asm=self.asm       ,   causal=self.causal       ,   n_langs=self.n_langs       ,   max_position_embeddings=self.max_position_embeddings       ,   initializer_range=self.initializer_range       ,   summary_type=self.summary_type       ,   use_proj=self.use_proj       ,   num_labels=self.num_labels       ,   bos_token_id=self.bos_token_id       ,   )





    def 					_lowerCamelCase (	self				:   Optional[Any]       ,   __snake_case				:   int       ,   __snake_case				:   int       ,   __snake_case				:   List[Any]       ,   __snake_case				:   List[Any]       ,   __snake_case				:   List[str]       ,   __snake_case				:   List[str]       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Dict       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   XLMModel(config=__snake_case	)
       model.to(__snake_case	)
       model.eval()
       UpperCAmelCase_							:      Optional[int]						=   model(__snake_case       ,   lengths=__snake_case       ,   langs=__snake_case	)
       UpperCAmelCase_							:      Any						=   model(__snake_case       ,   langs=__snake_case	)
       UpperCAmelCase_							:      Any						=   model(__snake_case	)
       self.parent.assertEqual(result.last_hidden_state.shape       ,   (self.batch_size, self.seq_length, self.hidden_size)	)





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   Dict       ,   __snake_case				:   Any       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Dict       ,   __snake_case				:   str       ,   __snake_case				:   Tuple       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   int       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   XLMWithLMHeadModel(__snake_case	)
       model.to(__snake_case	)
       model.eval()

       UpperCAmelCase_							:      Optional[Any]						=   model(__snake_case       ,   token_type_ids=__snake_case       ,   labels=__snake_case	)
       self.parent.assertEqual(result.loss.shape       ,   ()	)
       self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.seq_length, self.vocab_size)	)





    def 					_lowerCamelCase (	self				:   Optional[int]       ,   __snake_case				:   Tuple       ,   __snake_case				:   Any       ,   __snake_case				:   Tuple       ,   __snake_case				:   List[str]       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   List[str]       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   str       ,   __snake_case				:   List[str]       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   XLMForQuestionAnsweringSimple(__snake_case	)
       model.to(__snake_case	)
       model.eval()

       UpperCAmelCase_							:      Optional[int]						=   model(__snake_case	)

       UpperCAmelCase_							:      Any						=   model(__snake_case       ,   start_positions=__snake_case       ,   end_positions=__snake_case	)
       UpperCAmelCase_							:      Optional[Any]						=   outputs
       self.parent.assertEqual(result.start_logits.shape       ,   (self.batch_size, self.seq_length)	)
       self.parent.assertEqual(result.end_logits.shape       ,   (self.batch_size, self.seq_length)	)





    def 					_lowerCamelCase (	self				:   Any       ,   __snake_case				:   int       ,   __snake_case				:   Dict       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   List[str]       ,   __snake_case				:   Tuple       ,   __snake_case				:   str       ,   __snake_case				:   Tuple       ,   __snake_case				:   int       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      List[str]						=   XLMForQuestionAnswering(__snake_case	)
       model.to(__snake_case	)
       model.eval()

       UpperCAmelCase_							:      Union[str, Any]						=   model(__snake_case	)

       UpperCAmelCase_							:      List[str]						=   model(
           __snake_case       ,   start_positions=__snake_case       ,   end_positions=__snake_case       ,   cls_index=__snake_case       ,   is_impossible=__snake_case       ,   p_mask=__snake_case       ,   )

       UpperCAmelCase_							:      Optional[Any]						=   model(
           __snake_case       ,   start_positions=__snake_case       ,   end_positions=__snake_case       ,   cls_index=__snake_case       ,   is_impossible=__snake_case       ,   )

       ((UpperCAmelCase_)		,      )							:      Union[str, Any]						=   result_with_labels.to_tuple()

       UpperCAmelCase_							:      Optional[int]						=   model(__snake_case       ,   start_positions=__snake_case       ,   end_positions=__snake_case	)

       ((UpperCAmelCase_)		,      )							:      str						=   result_with_labels.to_tuple()

       self.parent.assertEqual(result_with_labels.loss.shape       ,   ()	)
       self.parent.assertEqual(result.start_top_log_probs.shape       ,   (self.batch_size, model.config.start_n_top)	)
       self.parent.assertEqual(result.start_top_index.shape       ,   (self.batch_size, model.config.start_n_top)	)
       self.parent.assertEqual(
           result.end_top_log_probs.shape       ,   (self.batch_size, model.config.start_n_top * model.config.end_n_top)	)
       self.parent.assertEqual(
           result.end_top_index.shape       ,   (self.batch_size, model.config.start_n_top * model.config.end_n_top)	)
       self.parent.assertEqual(result.cls_logits.shape       ,   (self.batch_size,)	)





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   Dict       ,   __snake_case				:   str       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   Dict       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Dict       ,   __snake_case				:   List[str]       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   Any       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      List[str]						=   XLMForSequenceClassification(__snake_case	)
       model.to(__snake_case	)
       model.eval()

       UpperCAmelCase_							:      Union[str, Any]						=   model(__snake_case	)
       UpperCAmelCase_							:      Optional[int]						=   model(__snake_case       ,   labels=__snake_case	)
       self.parent.assertEqual(result.loss.shape       ,   ()	)
       self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.type_sequence_label_size)	)





    def 					_lowerCamelCase (	self				:   Any       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Any       ,   __snake_case				:   List[str]       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   Tuple       ,   __snake_case				:   str       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Tuple       ,   __snake_case				:   int       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   self.num_labels
       UpperCAmelCase_							:      Optional[int]						=   XLMForTokenClassification(__snake_case	)
       model.to(__snake_case	)
       model.eval()

       UpperCAmelCase_							:      List[str]						=   model(__snake_case       ,   attention_mask=__snake_case       ,   labels=__snake_case	)
       self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.seq_length, self.num_labels)	)





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   str       ,   __snake_case				:   List[str]       ,   __snake_case				:   Any       ,   __snake_case				:   Tuple       ,   __snake_case				:   Any       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Dict       ,   __snake_case				:   Dict       ,   __snake_case				:   Optional[int]       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   self.num_choices
       UpperCAmelCase_							:      int						=   XLMForMultipleChoice(config=__snake_case	)
       model.to(__snake_case	)
       model.eval()
       UpperCAmelCase_							:      int						=   input_ids.unsqueeze(1	).expand(-1       ,   self.num_choices       ,   -1	).contiguous()
       UpperCAmelCase_							:      int						=   token_type_ids.unsqueeze(1	).expand(-1       ,   self.num_choices       ,   -1	).contiguous()
       UpperCAmelCase_							:      List[Any]						=   input_mask.unsqueeze(1	).expand(-1       ,   self.num_choices       ,   -1	).contiguous()
       UpperCAmelCase_							:      Any						=   model(
           __snake_case       ,   attention_mask=__snake_case       ,   token_type_ids=__snake_case       ,   labels=__snake_case       ,   )
       self.parent.assertEqual(result.logits.shape       ,   (self.batch_size, self.num_choices)	)





    def 					_lowerCamelCase (	self				:   List[str]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   self.prepare_config_and_inputs()
       (
           (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      (
           UpperCAmelCase_
       )		,      
       )							:      Union[str, Any]						=   config_and_inputs
       UpperCAmelCase_							:      str						=   {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''lengths''': input_lengths}
       return config, inputs_dict





@require_torch
class  lowerCAmelCase__(    snake_case__    ,    snake_case__    ,    snake_case__    ,    unittest.TestCase						):






    '''simple docstring'''





    A_  :     Dict			= (
        (
            XLMModel,
            XLMWithLMHeadModel,
            XLMForQuestionAnswering,
            XLMForSequenceClassification,
            XLMForQuestionAnsweringSimple,
            XLMForTokenClassification,
            XLMForMultipleChoice,
        )
        if is_torch_available()
        else ()
    )
    A_  :     List[str]			= (
        (XLMWithLMHeadModel,) if is_torch_available() else ()
    )  # TODO (PVP): Check other models whether language generation is also applicable
    A_  :     Optional[int]			= (
        {
            'feature-extraction': XLMModel,
            'fill-mask': XLMWithLMHeadModel,
            'question-answering': XLMForQuestionAnsweringSimple,
            'text-classification': XLMForSequenceClassification,
            'text-generation': XLMWithLMHeadModel,
            'token-classification': XLMForTokenClassification,
            'zero-shot': XLMForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Optional[Any]       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   Optional[Any]	):





       '''simple docstring'''
       if (
           pipeline_test_casse_name == "QAPipelineTests"
           and tokenizer_name is not None
           and not tokenizer_name.endswith('''Fast'''	)
       ):
          # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
          # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
          # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
          return True

       return False





    def 					_lowerCamelCase (	self				:   Union[str, Any]       ,   __snake_case				:   Any       ,   __snake_case				:   List[Any]       ,   __snake_case				:   str=False	):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   super()._prepare_for_class(__snake_case       ,   __snake_case       ,   return_labels=__snake_case	)

       if return_labels:
          if model_class.__name__ == "XLMForQuestionAnswering":
             UpperCAmelCase_							:      List[str]						=   torch.zeros(
                 self.model_tester.batch_size       ,   dtype=torch.long       ,   device=__snake_case	)
             UpperCAmelCase_							:      Dict						=   torch.zeros(
                 self.model_tester.batch_size       ,   dtype=torch.long       ,   device=__snake_case	)

       return inputs_dict





    def 					_lowerCamelCase (	self				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   XLMModelTester(self	)
       UpperCAmelCase_							:      List[Any]						=   ConfigTester(self       ,   config_class=__snake_case       ,   emb_dim=37	)





    def 					_lowerCamelCase (	self				:   Union[str, Any]	):





       '''simple docstring'''
       self.config_tester.run_common_tests()





    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_model(*__snake_case	)





    def 					_lowerCamelCase (	self				:   Union[str, Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Tuple						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_lm_head(*__snake_case	)





    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_simple_qa(*__snake_case	)





    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_qa(*__snake_case	)





    def 					_lowerCamelCase (	self				:   List[str]	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_sequence_classif(*__snake_case	)





    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_token_classif(*__snake_case	)





    def 					_lowerCamelCase (	self				:   str	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[int]						=   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_xlm_for_multiple_choice(*__snake_case	)





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   str       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   List[Any]       ,   __snake_case				:   str       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Union[str, Any]=False       ,   __snake_case				:   Optional[Any]=1	):





       '''simple docstring'''
       self.assertIsInstance(__snake_case       ,   __snake_case	)
       self.assertListEqual(
           [isinstance(__snake_case       ,   __snake_case	) for iter_attentions in attentions]       ,   [True] * len(__snake_case	)	)
       self.assertEqual(len(__snake_case	)       ,   (max_length - min_length) * num_beam_groups	)

       for idx, iter_attentions in enumerate(__snake_case	):
          # adds PAD dummy token
          UpperCAmelCase_							:      Dict						=   min_length + idx + 1
          UpperCAmelCase_							:      List[Any]						=   min_length + idx + 1

          UpperCAmelCase_							:      Optional[int]						=   (
              batch_size * num_beam_groups,
              config.num_attention_heads,
              tgt_len,
              src_len,
          )
          # check attn size
          self.assertListEqual(
              [layer_attention.shape for layer_attention in iter_attentions]       ,   [expected_shape] * len(__snake_case	)	)





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   List[str]       ,   __snake_case				:   int       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Dict       ,   __snake_case				:   Any       ,   __snake_case				:   Optional[Any]=False       ,   __snake_case				:   str=1	):





       '''simple docstring'''
       self.assertIsInstance(__snake_case       ,   __snake_case	)
       self.assertListEqual(
           [isinstance(__snake_case       ,   __snake_case	) for iter_hidden_states in hidden_states]       ,   [True] * len(__snake_case	)       ,   )
       self.assertEqual(len(__snake_case	)       ,   (max_length - min_length) * num_beam_groups	)

       for idx, iter_hidden_states in enumerate(__snake_case	):
          # adds PAD dummy token
          UpperCAmelCase_							:      str						=   min_length + idx + 1
          UpperCAmelCase_							:      int						=   (batch_size * num_beam_groups, seq_len, config.hidden_size)
          # check hidden size
          self.assertListEqual(
              [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]       ,   [expected_shape] * len(__snake_case	)       ,   )
       pass





    @slow
    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          UpperCAmelCase_							:      Any						=   XLMModel.from_pretrained(__snake_case	)
          self.assertIsNotNone(__snake_case	)





@require_torch
class  lowerCAmelCase__(    unittest.TestCase						):






    '''simple docstring'''





    @slow
    def 					_lowerCamelCase (	self				:   str	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[int]						=   XLMWithLMHeadModel.from_pretrained('''xlm-mlm-en-2048'''	)
       model.to(__snake_case	)
       UpperCAmelCase_							:      str						=   torch.tensor([[14, 447]]       ,   dtype=torch.long       ,   device=__snake_case	)  # the president
       UpperCAmelCase_							:      Union[str, Any]						=   [
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
           14,
           447,
       ]  # the president the president the president the president the president the president the president the president the president the president
       # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
       UpperCAmelCase_							:      Dict						=   model.generate(__snake_case       ,   do_sample=__snake_case	)
       self.assertListEqual(output_ids[0].cpu().numpy().tolist()       ,   __snake_case	)