


from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
   from fairscale.optim import OSS


_snake_case          =							logging.get_logger(__name__)

_snake_case          =							{
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    "constant": get_constant_schedule,
    "constant_w_warmup": get_constant_schedule_with_warmup,
}



class   _a      (      SCREAMING_SNAKE_CASE_ ):
    def __init__( self							:					str							,					SCREAMING_SNAKE_CASE__							:					Dict=None							,					SCREAMING_SNAKE_CASE__							:					Optional[Any]=None							,					*SCREAMING_SNAKE_CASE__							:					Dict							,					**SCREAMING_SNAKE_CASE__							:					List[str]				):
          super().__init__(*SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__				)

          if config is None:
                assert isinstance(self.model							,					SCREAMING_SNAKE_CASE__				), (
                    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
                    F' {self.model.__class__}'
                )
                lowerCamelCase__           =		self.model.config
          else:
                lowerCamelCase__           =		config

          lowerCamelCase__           =		data_args
          lowerCamelCase__           =		self.config.tgt_vocab_size if isinstance(self.config							,					SCREAMING_SNAKE_CASE__				) else self.config.vocab_size

          if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
                assert self.config.pad_token_id is not None, (
                    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
                    " calculation or doing label smoothing."
                )

          if self.config.pad_token_id is None and self.config.eos_token_id is not None:
                logger.warning(
                    F'The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'
                    ' padding..'				)

          if self.args.label_smoothing == 0:
                lowerCamelCase__           =		torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id				)
          else:
                # dynamically import label_smoothed_nll_loss
                from utils import label_smoothed_nll_loss

                lowerCamelCase__           =		label_smoothed_nll_loss
    def      _UpperCamelCase       ( self							:					str							,					SCREAMING_SNAKE_CASE__							:					int				):
          if self.optimizer is None:
                lowerCamelCase__           =		['bias', 'LayerNorm.weight']
                lowerCamelCase__           =		[
                    {
                        'params': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay				)],
                        'weight_decay': self.args.weight_decay,
                    },
                    {
                        'params': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay				)],
                        'weight_decay': 0.0,
                    },
                ]
                lowerCamelCase__           =		Adafactor if self.args.adafactor else AdamW
                if self.args.adafactor:
                      lowerCamelCase__           =		Adafactor
                      lowerCamelCase__           =		{'scale_parameter': False, 'relative_step': False}
                else:
                      lowerCamelCase__           =		AdamW
                      lowerCamelCase__           =		{
                          'betas': (self.args.adam_betaa, self.args.adam_betaa),
                          'eps': self.args.adam_epsilon,
                      }
                lowerCamelCase__           =		self.args.learning_rate
                if self.sharded_ddp:
                      lowerCamelCase__           =		OSS(
                          params=SCREAMING_SNAKE_CASE__							,					optim=SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__							,					)
                else:
                      lowerCamelCase__           =		optimizer_cls(SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__				)

          if self.lr_scheduler is None:
                lowerCamelCase__           =		self._get_lr_scheduler(SCREAMING_SNAKE_CASE__				)
          else:  # ignoring --lr_scheduler
                logger.warning('scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'				)
    def      _UpperCamelCase       ( self							:					Any							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]				):
          lowerCamelCase__           =		arg_to_scheduler[self.args.lr_scheduler]
          if self.args.lr_scheduler == "constant":
                lowerCamelCase__           =		schedule_func(self.optimizer				)
          elif self.args.lr_scheduler == "constant_w_warmup":
                lowerCamelCase__           =		schedule_func(self.optimizer							,					num_warmup_steps=self.args.warmup_steps				)
          else:
                lowerCamelCase__           =		schedule_func(
                    self.optimizer							,					num_warmup_steps=self.args.warmup_steps							,					num_training_steps=SCREAMING_SNAKE_CASE__				)
          return scheduler
    def      _UpperCamelCase       ( self							:					Any				):
          if isinstance(self.train_dataset							,					torch.utils.data.IterableDataset				):
                return None
          elif is_torch_tpu_available():
                return get_tpu_sampler(self.train_dataset				)
          else:
                if self.args.sortish_sampler:
                      self.train_dataset.make_sortish_sampler(
                          self.args.per_device_train_batch_size							,					distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)							,					)

                return (
                    RandomSampler(self.train_dataset				)
                    if self.args.local_rank == -1
                    else DistributedSampler(self.train_dataset				)
                )
    def      _UpperCamelCase       ( self							:					str							,					SCREAMING_SNAKE_CASE__							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					List[Any]							,					SCREAMING_SNAKE_CASE__							:					Tuple				):
          if self.args.label_smoothing == 0:
                if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
                      # force training to ignore pad token
                      lowerCamelCase__           =		model(**SCREAMING_SNAKE_CASE__							,					use_cache=SCREAMING_SNAKE_CASE__				)[0]
                      lowerCamelCase__           =		self.loss_fn(logits.view(-1							,					logits.shape[-1]				)							,					labels.view(-1				)				)
                else:
                      # compute usual loss via models
                      lowerCamelCase__    ,  lowerCamelCase__           =		model(**SCREAMING_SNAKE_CASE__							,					labels=SCREAMING_SNAKE_CASE__							,					use_cache=SCREAMING_SNAKE_CASE__				)[:2]
          else:
                # compute label smoothed loss
                lowerCamelCase__           =		model(**SCREAMING_SNAKE_CASE__							,					use_cache=SCREAMING_SNAKE_CASE__				)[0]
                lowerCamelCase__           =		torch.nn.functional.log_softmax(SCREAMING_SNAKE_CASE__							,					dim=-1				)
                lowerCamelCase__    ,  lowerCamelCase__           =		self.loss_fn(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__							,					self.args.label_smoothing							,					ignore_index=self.config.pad_token_id				)
          return loss, logits
    def      _UpperCamelCase       ( self							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					int							,					SCREAMING_SNAKE_CASE__							:					int				):
          lowerCamelCase__           =		inputs.pop('labels'				)
          lowerCamelCase__    ,  lowerCamelCase__           =		self._compute_loss(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)
          return loss
    def      _UpperCamelCase       ( self							:					str							,					SCREAMING_SNAKE_CASE__							:					nn.Module							,					SCREAMING_SNAKE_CASE__							:					Dict[str, Union[torch.Tensor, Any]]							,					SCREAMING_SNAKE_CASE__							:					bool							,					SCREAMING_SNAKE_CASE__							:					Optional[List[str]] = None							,					):
          lowerCamelCase__           =		self._prepare_inputs(SCREAMING_SNAKE_CASE__				)

          lowerCamelCase__           =		{
              'max_length': self.data_args.val_max_target_length
              if self.data_args is not None
              else self.config.max_length,
              'num_beams': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
          }

          if self.args.predict_with_generate and not self.args.prediction_loss_only:
                lowerCamelCase__           =		self.model.generate(
                    inputs['input_ids']							,					attention_mask=inputs['attention_mask']							,					**SCREAMING_SNAKE_CASE__							,					)
                # in case the batch is shorter than max length, the output should be padded
                if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
                      lowerCamelCase__           =		self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE__							,					gen_kwargs['max_length']				)

          lowerCamelCase__           =		inputs.pop('labels'				)
          with torch.no_grad():
                # compute loss on predict data
                lowerCamelCase__    ,  lowerCamelCase__           =		self._compute_loss(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)

          lowerCamelCase__           =		loss.mean().detach()
          if self.args.prediction_loss_only:
                return (loss, None, None)

          lowerCamelCase__           =		generated_tokens if self.args.predict_with_generate else logits

          if labels.shape[-1] < gen_kwargs["max_length"]:
                lowerCamelCase__           =		self._pad_tensors_to_max_len(SCREAMING_SNAKE_CASE__							,					gen_kwargs['max_length']				)

          return (loss, logits, labels)


    def      _UpperCamelCase       ( self							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					List[Any]							,					SCREAMING_SNAKE_CASE__							:					Tuple				):
          # If PAD token is not defined at least EOS token has to be defined
          lowerCamelCase__           =		self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

          if pad_token_id is None:
                raise ValueError(
                    'Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'
                    F' padded to `max_length`={max_length}'				)

          lowerCamelCase__           =		pad_token_id * torch.ones(
              (tensor.shape[0], max_length)							,					dtype=tensor.dtype							,					device=tensor.device				)
          lowerCamelCase__           =		tensor
          return padded_tensor







"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case          =							logging.get_logger(__name__)

_snake_case          =							{
    "microsoft/swinv2-tiny-patch4-window8-256": (
        "https://huggingface.co/microsoft/swinv2-tiny-patch4-window8-256/resolve/main/config.json"
    ),
}



class   _a      (      SCREAMING_SNAKE_CASE_ ):
    a_    :				Union[str, Any]             =    'swinv2'

    a_    :				Optional[int]             =    {
        'num_attention_heads': 'num_heads',
        'num_hidden_layers': 'num_layers',
    }
    def __init__( self							:					List[str]							,					SCREAMING_SNAKE_CASE__							:					int=2_24							,					SCREAMING_SNAKE_CASE__							:					str=4							,					SCREAMING_SNAKE_CASE__							:					List[Any]=3							,					SCREAMING_SNAKE_CASE__							:					str=96							,					SCREAMING_SNAKE_CASE__							:					Dict=[2, 2, 6, 2]							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]=[3, 6, 12, 24]							,					SCREAMING_SNAKE_CASE__							:					Tuple=7							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]=4.0							,					SCREAMING_SNAKE_CASE__							:					List[str]=True							,					SCREAMING_SNAKE_CASE__							:					List[Any]=0.0							,					SCREAMING_SNAKE_CASE__							:					Optional[int]=0.0							,					SCREAMING_SNAKE_CASE__							:					Dict=0.1							,					SCREAMING_SNAKE_CASE__							:					Tuple="gelu"							,					SCREAMING_SNAKE_CASE__							:					Tuple=False							,					SCREAMING_SNAKE_CASE__							:					Any=0.02							,					SCREAMING_SNAKE_CASE__							:					List[Any]=1e-5							,					SCREAMING_SNAKE_CASE__							:					int=32							,					**SCREAMING_SNAKE_CASE__							:					List[str]							,					):
          super().__init__(**SCREAMING_SNAKE_CASE__				)

          lowerCamelCase__           =		image_size
          lowerCamelCase__           =		patch_size
          lowerCamelCase__           =		num_channels
          lowerCamelCase__           =		embed_dim
          lowerCamelCase__           =		depths
          lowerCamelCase__           =		len(SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		num_heads
          lowerCamelCase__           =		window_size
          lowerCamelCase__           =		mlp_ratio
          lowerCamelCase__           =		qkv_bias
          lowerCamelCase__           =		hidden_dropout_prob
          lowerCamelCase__           =		attention_probs_dropout_prob
          lowerCamelCase__           =		drop_path_rate
          lowerCamelCase__           =		hidden_act
          lowerCamelCase__           =		use_absolute_embeddings
          lowerCamelCase__           =		layer_norm_eps
          lowerCamelCase__           =		initializer_range
          lowerCamelCase__           =		encoder_stride
          # we set the hidden_size attribute in order to make Swinv2 work with VisionEncoderDecoderModel
          # this indicates the channel dimension after the last stage of the model
          lowerCamelCase__           =		int(embed_dim * 2 ** (len(SCREAMING_SNAKE_CASE__				) - 1)				)
          lowerCamelCase__           =		(0, 0, 0, 0)



