


import argparse
import glob
import logging
import os
import time
from argparse import Namespace

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from torch.utils.data import DataLoader, TensorDataset

from transformers import glue_compute_metrics as compute_metrics
from transformers import glue_convert_examples_to_features as convert_examples_to_features
from transformers import glue_output_modes, glue_tasks_num_labels
from transformers import glue_processors as processors


__UpperCamelCase				:  Union[str, Any]      					=							logging.getLogger(__name__)





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     Tuple			= 'sequence-classification'





    def __init__(	self				:   Optional[Any]       ,   __snake_case				:   Optional[int]	):





       '''simple docstring'''
       if type(__snake_case	) == dict:
          UpperCAmelCase_							:      Dict						=   Namespace(**__snake_case	)
       UpperCAmelCase_							:      Union[str, Any]						=   glue_output_modes[hparams.task]
       UpperCAmelCase_							:      Dict						=   glue_tasks_num_labels[hparams.task]

       super().__init__(__snake_case       ,   __snake_case       ,   self.mode	)





    def 					_lowerCamelCase (	self				:   List[Any]       ,   **__snake_case				:   Any	):





       '''simple docstring'''
       return self.model(**__snake_case	)





    def 					_lowerCamelCase (	self				:   Any       ,   __snake_case				:   str       ,   __snake_case				:   int	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   {'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}

       if self.config.model_type not in ["distilbert", "bart"]:
          UpperCAmelCase_							:      Optional[Any]						=   batch[2] if self.config.model_type in ['''bert''', '''xlnet''', '''albert'''] else None

       UpperCAmelCase_							:      Optional[int]						=   self(**__snake_case	)
       UpperCAmelCase_							:      int						=   outputs[0]

       UpperCAmelCase_							:      Dict						=   self.trainer.lr_schedulers[0]['''scheduler''']
       UpperCAmelCase_							:      Optional[Any]						=   {'''loss''': loss, '''rate''': lr_scheduler.get_last_lr()[-1]}
       return {"loss": loss, "log": tensorboard_logs}





    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   self.hparams
       UpperCAmelCase_							:      Dict						=   processors[args.task]()
       UpperCAmelCase_							:      int						=   processor.get_labels()

       for mode in ["train", "dev"]:
          UpperCAmelCase_							:      str						=   self._feature_file(__snake_case	)
          if os.path.exists(__snake_case	) and not args.overwrite_cache:
             logger.info('''Loading features from cached file %s'''       ,   __snake_case	)
          else:
             logger.info('''Creating features from dataset file at %s'''       ,   args.data_dir	)
             UpperCAmelCase_							:      List[str]						=   (
                 processor.get_dev_examples(args.data_dir	)
                 if mode == '''dev'''
                 else processor.get_train_examples(args.data_dir	)
             )
             UpperCAmelCase_							:      List[str]						=   convert_examples_to_features(
                 __snake_case       ,   self.tokenizer       ,   max_length=args.max_seq_length       ,   label_list=self.labels       ,   output_mode=args.glue_output_mode       ,   )
             logger.info('''Saving features into cached file %s'''       ,   __snake_case	)
             torch.save(__snake_case       ,   __snake_case	)





    def 					_lowerCamelCase (	self				:   Optional[Any]       ,   __snake_case				:   str       ,   __snake_case				:   int       ,   __snake_case				:   bool = False	):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   '''dev''' if mode == '''test''' else mode

       UpperCAmelCase_							:      Any						=   self._feature_file(__snake_case	)
       logger.info('''Loading features from cached file %s'''       ,   __snake_case	)
       UpperCAmelCase_							:      Dict						=   torch.load(__snake_case	)
       UpperCAmelCase_							:      List[Any]						=   torch.tensor([f.input_ids for f in features]       ,   dtype=torch.long	)
       UpperCAmelCase_							:      List[str]						=   torch.tensor([f.attention_mask for f in features]       ,   dtype=torch.long	)
       UpperCAmelCase_							:      List[Any]						=   torch.tensor([f.token_type_ids for f in features]       ,   dtype=torch.long	)
       if self.hparams.glue_output_mode == "classification":
          UpperCAmelCase_							:      List[Any]						=   torch.tensor([f.label for f in features]       ,   dtype=torch.long	)
       elif self.hparams.glue_output_mode == "regression":
          UpperCAmelCase_							:      Union[str, Any]						=   torch.tensor([f.label for f in features]       ,   dtype=torch.float	)

       return DataLoader(
           TensorDataset(__snake_case       ,   __snake_case       ,   __snake_case       ,   __snake_case	)       ,   batch_size=__snake_case       ,   shuffle=__snake_case       ,   )





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   Union[str, Any]       ,   __snake_case				:   str	):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   {'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}

       if self.config.model_type not in ["distilbert", "bart"]:
          UpperCAmelCase_							:      Any						=   batch[2] if self.config.model_type in ['''bert''', '''xlnet''', '''albert'''] else None

       UpperCAmelCase_							:      Tuple						=   self(**__snake_case	)
       UpperCAmelCase_							:      str						=   outputs[:2]
       UpperCAmelCase_							:      List[Any]						=   logits.detach().cpu().numpy()
       UpperCAmelCase_							:      Union[str, Any]						=   inputs['''labels'''].detach().cpu().numpy()

       return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}





    def 					_lowerCamelCase (	self				:   Optional[Any]       ,   __snake_case				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   torch.stack([x['''val_loss'''] for x in outputs]	).mean().detach().cpu().item()
       UpperCAmelCase_							:      Optional[Any]						=   np.concatenate([x['''pred'''] for x in outputs]       ,   axis=0	)

       if self.hparams.glue_output_mode == "classification":
          UpperCAmelCase_							:      Any						=   np.argmax(__snake_case       ,   axis=1	)
       elif self.hparams.glue_output_mode == "regression":
          UpperCAmelCase_							:      Union[str, Any]						=   np.squeeze(__snake_case	)

       UpperCAmelCase_							:      Optional[int]						=   np.concatenate([x['''target'''] for x in outputs]       ,   axis=0	)
       UpperCAmelCase_							:      str						=   [[] for _ in range(out_label_ids.shape[0]	)]
       UpperCAmelCase_							:      str						=   [[] for _ in range(out_label_ids.shape[0]	)]

       UpperCAmelCase_							:      Union[str, Any]						=   {**{'''val_loss''': val_loss_mean}, **compute_metrics(self.hparams.task       ,   __snake_case       ,   __snake_case	)}

       UpperCAmelCase_							:      List[str]						=   dict(results.items()	)
       UpperCAmelCase_							:      Optional[int]						=   results
       return ret, preds_list, out_label_list





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   list	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   self._eval_end(__snake_case	)
       UpperCAmelCase_							:      Dict						=   ret['''log''']
       return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   str	):





       '''simple docstring'''
       UpperCAmelCase_							:      Any						=   self._eval_end(__snake_case	)
       UpperCAmelCase_							:      List[str]						=   ret['''log''']
       # `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
       return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}





    @staticmethod
    def 					_lowerCamelCase (	__snake_case				:   str       ,   __snake_case				:   Any	):





       '''simple docstring'''
       BaseTransformer.add_model_specific_args(__snake_case       ,   __snake_case	)
       parser.add_argument(
           '''--max_seq_length'''       ,   default=128       ,   type=__snake_case       ,   help=(
               '''The maximum total input sequence length after tokenization. Sequences longer '''
               '''than this will be truncated, sequences shorter will be padded.'''
           )       ,   )

       parser.add_argument(
           '''--task'''       ,   default=''''''       ,   type=__snake_case       ,   required=__snake_case       ,   help='''The GLUE task to run'''       ,   )
       parser.add_argument(
           '''--gpus'''       ,   default=0       ,   type=__snake_case       ,   help='''The number of GPUs allocated for this, it is by default 0 meaning none'''       ,   )

       parser.add_argument(
           '''--overwrite_cache'''       ,   action='''store_true'''       ,   help='''Overwrite the cached training and evaluation sets'''	)

       return parser





def      snake_case_						( ):
   UpperCAmelCase_							:      Any						=   argparse.ArgumentParser()
   add_generic_args(__lowercase			,       os.getcwd()       )
   UpperCAmelCase_							:      Tuple						=   GLUETransformer.add_model_specific_args(__lowercase			,       os.getcwd()       )
   UpperCAmelCase_							:      Union[str, Any]						=   parser.parse_args()

   # If output_dir not provided, a folder will be generated in pwd
   if args.output_dir is None:
      UpperCAmelCase_							:      int						=   os.path.join(
          '''./results'''			,       F'''{args.task}_{time.strftime('%Y%m%d_%H%M%S'       )}'''			,       )
      os.makedirs(args.output_dir       )

   UpperCAmelCase_							:      str						=   GLUETransformer(__lowercase       )
   UpperCAmelCase_							:      str						=   generic_train(__lowercase			,       __lowercase       )

   # Optionally, predict on dev set and write to output_dir
   if args.do_predict:
      UpperCAmelCase_							:      List[Any]						=   sorted(glob.glob(os.path.join(args.output_dir			,       '''checkpoint-epoch=*.ckpt'''       )			,       recursive=__lowercase       )       )
      UpperCAmelCase_							:      Optional[int]						=   model.load_from_checkpoint(checkpoints[-1]       )
      return trainer.test(__lowercase       )


if __name__ == "__main__":
       main()








import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase				:  Tuple      					=							logging.get_logger(__name__)





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     Union[str, Any]			= 'encoder-decoder'
    A_  :     Optional[int]			= True





    def __init__(	self				:   Dict       ,   **__snake_case				:   Union[str, Any]	):





       '''simple docstring'''
       super().__init__(**__snake_case	)
       assert (
           "encoder" in kwargs and "decoder" in kwargs
       ), "Config has to be initialized with encoder and decoder config"
       UpperCAmelCase_							:      int						=   kwargs.pop('''encoder'''	)
       UpperCAmelCase_							:      List[Any]						=   encoder_config.pop('''model_type'''	)
       UpperCAmelCase_							:      int						=   kwargs.pop('''decoder'''	)
       UpperCAmelCase_							:      Union[str, Any]						=   decoder_config.pop('''model_type'''	)

       from ..auto.configuration_auto import AutoConfig

       UpperCAmelCase_							:      Optional[int]						=   AutoConfig.for_model(__snake_case       ,   **__snake_case	)
       UpperCAmelCase_							:      Any						=   AutoConfig.for_model(__snake_case       ,   **__snake_case	)
       UpperCAmelCase_							:      List[Any]						=   True





    @classmethod
    def 					_lowerCamelCase (	cls				:   Optional[Any]       ,   __snake_case				:   PretrainedConfig       ,   __snake_case				:   PretrainedConfig       ,   **__snake_case				:   Union[str, Any]	):





       '''simple docstring'''
       logger.info('''Set `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config'''	)
       UpperCAmelCase_							:      Dict						=   True
       UpperCAmelCase_							:      str						=   True

       return cls(encoder=encoder_config.to_dict()       ,   decoder=decoder_config.to_dict()       ,   **__snake_case	)





    def 					_lowerCamelCase (	self				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   copy.deepcopy(self.__dict__	)
       UpperCAmelCase_							:      Tuple						=   self.encoder.to_dict()
       UpperCAmelCase_							:      Tuple						=   self.decoder.to_dict()
       UpperCAmelCase_							:      Tuple						=   self.__class__.model_type
       return output