


from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


snake_case							: Dict      =   {
    'configuration_autoformer': [
        'AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'AutoformerConfig',
    ],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						snake_case							: Optional[int]      =   [
						    'AUTOFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'AutoformerForPrediction',
						    'AutoformerModel',
						    'AutoformerPreTrainedModel',
						]


if TYPE_CHECKING:
						from .configuration_autoformer import (
						    AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
						    AutoformerConfig,
						)

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_autoformer import (
												    AUTOFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
												    AutoformerForPrediction,
												    AutoformerModel,
												    AutoformerPreTrainedModel,
												)

else:
						import sys

						snake_case							: Tuple      =   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)








from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
						from fairscale.optim import OSS


snake_case							: Any      =   logging.get_logger(__name__)

snake_case							: Tuple      =   {
    '''linear''': get_linear_schedule_with_warmup,
    '''cosine''': get_cosine_schedule_with_warmup,
    '''cosine_w_restarts''': get_cosine_with_hard_restarts_schedule_with_warmup,
    '''polynomial''': get_polynomial_decay_schedule_with_warmup,
    '''constant''': get_constant_schedule,
    '''constant_w_warmup''': get_constant_schedule_with_warmup,
}







class  snake_case_     (lowerCamelCase_   ):
					def __init__(   self     :str    ,__snake_case     :Dict=None    ,__snake_case     :int=None    ,*__snake_case     :str    ,**__snake_case     :Union[str, Any]   )      ->					Tuple:
												super().__init__(*__snake_case    ,**__snake_case   )

												if config is None:
																			assert isinstance(self.model    ,__snake_case   ), (
																			    "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
																			    F' {self.model.__class__}'
																			)
																			a__    =  self.model.config
												else:
																			a__    =  config

												a__    =  data_args
												a__    =  self.config.tgt_vocab_size if isinstance(self.config    ,__snake_case   ) else self.config.vocab_size

												if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
																			assert self.config.pad_token_id is not None, (
																			    "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
																			    " calculation or doing label smoothing."
																			)

												if self.config.pad_token_id is None and self.config.eos_token_id is not None:
																			logger.warning(
																			    F'The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for'
																			    ' padding..'   )

												if self.args.label_smoothing == 0:
																			a__    =  torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id   )
												else:
																			# dynamically import label_smoothed_nll_loss
																			from utils import label_smoothed_nll_loss

																			a__    =  label_smoothed_nll_loss
					def 				lowerCamelCase__(   self     :Optional[Any]    ,__snake_case     :int   )      ->					Tuple:
												if self.optimizer is None:
																			a__    =  ['bias', 'LayerNorm.weight']
																			a__    =  [
																			    {
																			        'params': [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay   )],
																			        'weight_decay': self.args.weight_decay,
																			    },
																			    {
																			        'params': [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay   )],
																			        'weight_decay': 0.0,
																			    },
																			]
																			a__    =  Adafactor if self.args.adafactor else AdamW
																			if self.args.adafactor:
																										a__    =  Adafactor
																										a__    =  {'scale_parameter': False, 'relative_step': False}
																			else:
																										a__    =  AdamW
																										a__    =  {
																										    'betas': (self.args.adam_betaa, self.args.adam_betaa),
																										    'eps': self.args.adam_epsilon,
																										}
																			a__    =  self.args.learning_rate
																			if self.sharded_ddp:
																										a__    =  OSS(
																										    params=__snake_case    ,optim=__snake_case    ,**__snake_case    ,)
																			else:
																										a__    =  optimizer_cls(__snake_case    ,**__snake_case   )

												if self.lr_scheduler is None:
																			a__    =  self._get_lr_scheduler(__snake_case   )
												else:  # ignoring --lr_scheduler
																			logger.warning('scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored.'   )
					def 				lowerCamelCase__(   self     :Dict    ,__snake_case     :List[str]   )      ->					Union[str, Any]:
												a__    =  arg_to_scheduler[self.args.lr_scheduler]
												if self.args.lr_scheduler == "constant":
																			a__    =  schedule_func(self.optimizer   )
												elif self.args.lr_scheduler == "constant_w_warmup":
																			a__    =  schedule_func(self.optimizer    ,num_warmup_steps=self.args.warmup_steps   )
												else:
																			a__    =  schedule_func(
																			    self.optimizer    ,num_warmup_steps=self.args.warmup_steps    ,num_training_steps=__snake_case   )
												return scheduler
					def 				lowerCamelCase__(   self     :Optional[Any]   )      ->					Optional[torch.utils.data.Sampler]:
												if isinstance(self.train_dataset    ,torch.utils.data.IterableDataset   ):
																			return None
												elif is_torch_tpu_available():
																			return get_tpu_sampler(self.train_dataset   )
												else:
																			if self.args.sortish_sampler:
																										self.train_dataset.make_sortish_sampler(
																										    self.args.per_device_train_batch_size    ,distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED)    ,)

																			return (
																			    RandomSampler(self.train_dataset   )
																			    if self.args.local_rank == -1
																			    else DistributedSampler(self.train_dataset   )
																			)
					def 				lowerCamelCase__(   self     :str    ,__snake_case     :Optional[int]    ,__snake_case     :List[Any]    ,__snake_case     :Any   )      ->					Optional[Any]:
												if self.args.label_smoothing == 0:
																			if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
																										# force training to ignore pad token
																										a__    =  model(**__snake_case    ,use_cache=__snake_case   )[0]
																										a__    =  self.loss_fn(logits.view(-1    ,logits.shape[-1]   )    ,labels.view(-1   )   )
																			else:
																										# compute usual loss via models
																										a__   ,      a__    =  model(**__snake_case    ,labels=__snake_case    ,use_cache=__snake_case   )[:2]
												else:
																			# compute label smoothed loss
																			a__    =  model(**__snake_case    ,use_cache=__snake_case   )[0]
																			a__    =  torch.nn.functional.log_softmax(__snake_case    ,dim=-1   )
																			a__   ,      a__    =  self.loss_fn(__snake_case    ,__snake_case    ,self.args.label_smoothing    ,ignore_index=self.config.pad_token_id   )
												return loss, logits
					def 				lowerCamelCase__(   self     :List[Any]    ,__snake_case     :Dict    ,__snake_case     :Optional[int]   )      ->					Any:
												a__    =  inputs.pop('labels'   )
												a__   ,      a__    =  self._compute_loss(__snake_case    ,__snake_case    ,__snake_case   )
												return loss
					def 				lowerCamelCase__(   self     :Optional[Any]    ,__snake_case     :nn.Module    ,__snake_case     :Dict[str, Union[torch.Tensor, Any]]    ,__snake_case     :bool    ,__snake_case     :Optional[List[str]] = None    ,)      ->					Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]:
												a__    =  self._prepare_inputs(__snake_case   )

												a__    =  {
												    'max_length': self.data_args.val_max_target_length
												    if self.data_args is not None
												    else self.config.max_length,
												    'num_beams': self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
												}

												if self.args.predict_with_generate and not self.args.prediction_loss_only:
																			a__    =  self.model.generate(
																			    inputs['input_ids']    ,attention_mask=inputs['attention_mask']    ,**__snake_case    ,)
																			# in case the batch is shorter than max length, the output should be padded
																			if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
																										a__    =  self._pad_tensors_to_max_len(__snake_case    ,gen_kwargs['max_length']   )

												a__    =  inputs.pop('labels'   )
												with torch.no_grad():
																			# compute loss on predict data
																			a__   ,      a__    =  self._compute_loss(__snake_case    ,__snake_case    ,__snake_case   )

												a__    =  loss.mean().detach()
												if self.args.prediction_loss_only:
																			return (loss, None, None)

												a__    =  generated_tokens if self.args.predict_with_generate else logits

												if labels.shape[-1] < gen_kwargs["max_length"]:
																			a__    =  self._pad_tensors_to_max_len(__snake_case    ,gen_kwargs['max_length']   )

												return (loss, logits, labels)



					def 				lowerCamelCase__(   self     :List[str]    ,__snake_case     :Optional[Any]    ,__snake_case     :Union[str, Any]   )      ->					int:
												# If PAD token is not defined at least EOS token has to be defined
												a__    =  self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

												if pad_token_id is None:
																			raise ValueError(
																			    'Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be'
																			    F' padded to `max_length`={max_length}'   )

												a__    =  pad_token_id * torch.ones(
												    (tensor.shape[0], max_length)    ,dtype=tensor.dtype    ,device=tensor.device   )
												a__    =  tensor
												return padded_tensor




