





'''simple docstring'''




import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only
from utils_rag import save_json







def _lowerCAmelCase     (      _lowerCAmelCase    )->     Union[str, Any]:
 __UpperCAmelCase					=   filter(lambda _lowerCAmelCase    : p.requires_grad       ,    model.parameters()    )
 __UpperCAmelCase					=   sum([np.prod(p.size()    ) for p in model_parameters]    )
 return params


_A:					Dict							=				logging.getLogger(__name__)







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase    )->     Optional[int]:
 if metric == "rouge2":
  __UpperCAmelCase					=   '{val_avg_rouge2:.4f}-{step_count}'
 elif metric == "bleu":
  __UpperCAmelCase					=   '{val_avg_bleu:.4f}-{step_count}'
 elif metric == "em":
  __UpperCAmelCase					=   '{val_avg_em:.4f}-{step_count}'
 elif metric == "loss":
  __UpperCAmelCase					=   '{val_avg_loss:.4f}-{step_count}'
 else:
  raise NotImplementedError(
      F'seq2seq callbacks only support rouge2 and bleu, got {metric}, You can make your own by adding to this'
      ' function.'    )

 __UpperCAmelCase					=   ModelCheckpoint(
     dirpath=_lowerCAmelCase       ,    filename=_lowerCAmelCase       ,    monitor=F'val_{metric}'       ,    mode='max'       ,    save_top_k=1       ,    every_n_epochs=1       ,    )

 return checkpoint_callback







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase    )->     List[Any]:
 return EarlyStopping(
     monitor=F'val_{metric}'       ,    mode='min' if 'loss' in metric else 'max'       ,    patience=_lowerCAmelCase       ,    verbose=_lowerCAmelCase       ,    )






class UpperCAmelCase							(		pl.Callback							):



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     __UpperCAmelCase					=   {f'lr_group_{i}': param['lr'] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups					)}
     pl_module.logger.log_metrics(__A					)



    @rank_zero_only
    def 					__lowerCamelCase						(  self     ,			__A     ,			__A     ,			__A     ,			__A=True					):
     logger.info(f'***** {type_path} results at step {trainer.global_step:05d} *****'					)
     __UpperCAmelCase					=   trainer.callback_metrics
     trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ['log', 'progress_bar', 'preds']}					)
     # Log results
     __UpperCAmelCase					=   Path(pl_module.hparams.output_dir					)
     if type_path == "test":
      __UpperCAmelCase					=   od / 'test_results.txt'
      __UpperCAmelCase					=   od / 'test_generations.txt'
     else:
      # this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
      # If people want this it will be easy enough to add back.
      __UpperCAmelCase					=   od / f'{type_path}_results/{trainer.global_step:05d}.txt'
      __UpperCAmelCase					=   od / f'{type_path}_generations/{trainer.global_step:05d}.txt'
      results_file.parent.mkdir(exist_ok=__A					)
      generations_file.parent.mkdir(exist_ok=__A					)
     with open(__A     ,			'a+'					) as writer:
      for key in sorted(__A					):
       if key in ["log", "progress_bar", "preds"]:
        continue
       __UpperCAmelCase					=   metrics[key]
       if isinstance(__A     ,			torch.Tensor					):
        __UpperCAmelCase					=   val.item()
       __UpperCAmelCase					=   f'{key}: {val:.6f}\n'
       writer.write(__A					)

     if not save_generations:
      return

     if "preds" in metrics:
      __UpperCAmelCase					=   '\n'.join(metrics['preds']					)
      generations_file.open('w+'					).write(__A					)



    @rank_zero_only
    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     try:
      __UpperCAmelCase					=   pl_module.model.model.num_parameters()
     except AttributeError:
      __UpperCAmelCase					=   pl_module.model.num_parameters()

     __UpperCAmelCase					=   count_trainable_parameters(__A					)
     # mp stands for million parameters
     trainer.logger.log_metrics({'n_params': npars, 'mp': npars / 1E6, 'grad_mp': n_trainable_pars / 1E6}					)



    @rank_zero_only
    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     save_json(pl_module.metrics     ,			pl_module.metrics_save_path					)
     return self._write_logs(__A     ,			__A     ,			'test'					)




    @rank_zero_only
    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     save_json(pl_module.metrics     ,			pl_module.metrics_save_path					)
     # Uncommenting this will save val generations
     # return self._write_logs(trainer, pl_module, "valid")












'''simple docstring'''




import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


_A:					int							=				logging.getLogger(__name__)






class UpperCAmelCase							:



    def __init__(  self					):
     __UpperCAmelCase					=   False



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A     ,			__A     ,			__A					):
     if not self.initialized:
      __UpperCAmelCase					=   RagRetriever(
          __A     ,			question_encoder_tokenizer=__A     ,			generator_tokenizer=__A     ,			index=__A     ,			init_retrieval=__A     ,			)
      __UpperCAmelCase					=   True



    def 					__lowerCamelCase						(  self					):
     self.retriever.index.init_index()




    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     __UpperCAmelCase					,    __UpperCAmelCase					=   self.retriever._main_retrieve(__A     ,			__A					)
     return doc_ids, retrieved_doc_embeds






class UpperCAmelCase							(		UpperCAmelCase_							):



    def __init__(  self     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A=None					):
     if index is not None and index.is_initialized() and len(__A					) > 0:
      raise ValueError(
          'When using Ray for distributed fine-tuning, '
          'you\'ll need to provide the paths instead, '
          'as the dataset and the index are loaded '
          'separately. More info in examples/rag/use_own_knowledge_dataset.py '					)
     super().__init__(
         __A     ,			question_encoder_tokenizer=__A     ,			generator_tokenizer=__A     ,			index=__A     ,			init_retrieval=__A     ,			)
     __UpperCAmelCase					=   retrieval_workers
     if len(self.retrieval_workers					) > 0:
      ray.get(
          [
              worker.create_rag_retriever.remote(__A     ,			__A     ,			__A     ,			__A					)
              for worker in self.retrieval_workers
          ]					)



    def 					__lowerCamelCase						(  self					):
     logger.info('initializing retrieval'					)

     if len(self.retrieval_workers					) > 0:
      ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]					)
     else:
      # Non-distributed training. Load index into this same process.
      self.index.init_index()



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     if len(self.retrieval_workers					) > 0:
      # Select a random retrieval actor.
      __UpperCAmelCase					=   self.retrieval_workers[random.randint(0     ,			len(self.retrieval_workers					) - 1					)]
      __UpperCAmelCase					,    __UpperCAmelCase					=   ray.get(random_worker.retrieve.remote(__A     ,			__A					)					)
     else:
      __UpperCAmelCase					,    __UpperCAmelCase					=   self._main_retrieve(__A     ,			__A					)
     return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(__A					)



    @classmethod
    def 					__lowerCamelCase						(  cls     ,			__A     ,			__A=None     ,			**__A					):
     return super(__A     ,			cls					).get_tokenizers(__A     ,			__A     ,			**__A					)




    @classmethod
    def 					__lowerCamelCase						(  cls     ,			__A     ,			__A     ,			__A=None     ,			**__A					):
     __UpperCAmelCase					=   kwargs.pop('config'     ,			__A					) or RagConfig.from_pretrained(__A     ,			**__A					)
     __UpperCAmelCase					=   RagTokenizer.from_pretrained(__A     ,			config=__A					)
     __UpperCAmelCase					=   rag_tokenizer.question_encoder
     __UpperCAmelCase					=   rag_tokenizer.generator
     if indexed_dataset is not None:
      __UpperCAmelCase					=   'custom'
      __UpperCAmelCase					=   CustomHFIndex(config.retrieval_vector_size     ,			__A					)
     else:
      __UpperCAmelCase					=   cls._build_index(__A					)
     return cls(
         __A     ,			question_encoder_tokenizer=__A     ,			generator_tokenizer=__A     ,			retrieval_workers=__A     ,			index=__A     ,			)





