





'''simple docstring'''




from ..utils import DummyObject, requires_backends






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			str						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Union[str, Any]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Dict						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Union[str, Any]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Tuple						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Tuple						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Any						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Any						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			int						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			List[str]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			List[str]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Tuple						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			int						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Any						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Tuple						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Dict						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Tuple						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Optional[int]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			List[Any]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Union[str, Any]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Dict						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			List[str]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			int						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			int						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Optional[Any]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			List[Any]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Dict						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Tuple						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			List[str]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			str						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)






class UpperCAmelCase							(		metaclass=UpperCAmelCase_							):
    _A :			Union[str, Any]						   =       ["""sentencepiece"""]



    def __init__(  self     ,			*__A     ,			**__A					):
     requires_backends(self     ,			['sentencepiece']					)












'''simple docstring'''




import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


_A:					int							=				logging.getLogger(__name__)






class UpperCAmelCase							:



    def __init__(  self					):
     __UpperCAmelCase					=   False



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A     ,			__A     ,			__A					):
     if not self.initialized:
      __UpperCAmelCase					=   RagRetriever(
          __A     ,			question_encoder_tokenizer=__A     ,			generator_tokenizer=__A     ,			index=__A     ,			init_retrieval=__A     ,			)
      __UpperCAmelCase					=   True



    def 					__lowerCamelCase						(  self					):
     self.retriever.index.init_index()




    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     __UpperCAmelCase					,    __UpperCAmelCase					=   self.retriever._main_retrieve(__A     ,			__A					)
     return doc_ids, retrieved_doc_embeds






class UpperCAmelCase							(		UpperCAmelCase_							):



    def __init__(  self     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A=None					):
     if index is not None and index.is_initialized() and len(__A					) > 0:
      raise ValueError(
          'When using Ray for distributed fine-tuning, '
          'you\'ll need to provide the paths instead, '
          'as the dataset and the index are loaded '
          'separately. More info in examples/rag/use_own_knowledge_dataset.py '					)
     super().__init__(
         __A     ,			question_encoder_tokenizer=__A     ,			generator_tokenizer=__A     ,			index=__A     ,			init_retrieval=__A     ,			)
     __UpperCAmelCase					=   retrieval_workers
     if len(self.retrieval_workers					) > 0:
      ray.get(
          [
              worker.create_rag_retriever.remote(__A     ,			__A     ,			__A     ,			__A					)
              for worker in self.retrieval_workers
          ]					)



    def 					__lowerCamelCase						(  self					):
     logger.info('initializing retrieval'					)

     if len(self.retrieval_workers					) > 0:
      ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]					)
     else:
      # Non-distributed training. Load index into this same process.
      self.index.init_index()



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     if len(self.retrieval_workers					) > 0:
      # Select a random retrieval actor.
      __UpperCAmelCase					=   self.retrieval_workers[random.randint(0     ,			len(self.retrieval_workers					) - 1					)]
      __UpperCAmelCase					,    __UpperCAmelCase					=   ray.get(random_worker.retrieve.remote(__A     ,			__A					)					)
     else:
      __UpperCAmelCase					,    __UpperCAmelCase					=   self._main_retrieve(__A     ,			__A					)
     return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(__A					)



    @classmethod
    def 					__lowerCamelCase						(  cls     ,			__A     ,			__A=None     ,			**__A					):
     return super(__A     ,			cls					).get_tokenizers(__A     ,			__A     ,			**__A					)




    @classmethod
    def 					__lowerCamelCase						(  cls     ,			__A     ,			__A     ,			__A=None     ,			**__A					):
     __UpperCAmelCase					=   kwargs.pop('config'     ,			__A					) or RagConfig.from_pretrained(__A     ,			**__A					)
     __UpperCAmelCase					=   RagTokenizer.from_pretrained(__A     ,			config=__A					)
     __UpperCAmelCase					=   rag_tokenizer.question_encoder
     __UpperCAmelCase					=   rag_tokenizer.generator
     if indexed_dataset is not None:
      __UpperCAmelCase					=   'custom'
      __UpperCAmelCase					=   CustomHFIndex(config.retrieval_vector_size     ,			__A					)
     else:
      __UpperCAmelCase					=   cls._build_index(__A					)
     return cls(
         __A     ,			question_encoder_tokenizer=__A     ,			generator_tokenizer=__A     ,			retrieval_workers=__A     ,			index=__A     ,			)





