





'''simple docstring'''




import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin






class     UpperCAmelCase							(		UpperCAmelCase_							):
    _A :			Any						   =       """Speech2TextFeatureExtractor"""
    _A :			List[Any]						   =       """Speech2TextTokenizer"""



    def __init__(  self     ,			__A     ,			__A					):
     super().__init__(__A     ,			__A					)
     __UpperCAmelCase					=   self.feature_extractor
     __UpperCAmelCase					=   False



    def __call__(  self     ,			*__A     ,			**__A					):
     # For backward compatibility
     if self._in_target_context_manager:
      return self.current_processor(*__A     ,			**__A					)

     if "raw_speech" in kwargs:
      warnings.warn('Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'					)
      __UpperCAmelCase					=   kwargs.pop('raw_speech'					)
     else:
      __UpperCAmelCase					=   kwargs.pop('audio'     ,			__A					)
     __UpperCAmelCase					=   kwargs.pop('sampling_rate'     ,			__A					)
     __UpperCAmelCase					=   kwargs.pop('text'     ,			__A					)
     if len(__A					) > 0:
      __UpperCAmelCase					=   args[0]
      __UpperCAmelCase					=   args[1:]

     if audio is None and text is None:
      raise ValueError('You need to specify either an `audio` or `text` input to process.'					)

     if audio is not None:
      __UpperCAmelCase					=   self.feature_extractor(__A     ,			*__A     ,			sampling_rate=__A     ,			**__A					)
     if text is not None:
      __UpperCAmelCase					=   self.tokenizer(__A     ,			**__A					)

     if text is None:
      return inputs
     elif audio is None:
      return encodings
     else:
      __UpperCAmelCase					=   encodings['input_ids']
      return inputs



    def 					__lowerCamelCase						(  self     ,			*__A     ,			**__A					):
     return self.tokenizer.batch_decode(*__A     ,			**__A					)



    def 					__lowerCamelCase						(  self     ,			*__A     ,			**__A					):
     return self.tokenizer.decode(*__A     ,			**__A					)




    @contextmanager
    def 					__lowerCamelCase						(  self					):
     warnings.warn(
         '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
         'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
         'your audio inputs, or in a separate call.'					)
     __UpperCAmelCase					=   True
     __UpperCAmelCase					=   self.tokenizer
     yield
     __UpperCAmelCase					=   self.feature_extractor
     __UpperCAmelCase					=   False












'''simple docstring'''




import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


_A:					int							=				logging.getLogger(__name__)






class UpperCAmelCase							:



    def __init__(  self					):
     __UpperCAmelCase					=   False



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A     ,			__A     ,			__A					):
     if not self.initialized:
      __UpperCAmelCase					=   RagRetriever(
          __A     ,			question_encoder_tokenizer=__A     ,			generator_tokenizer=__A     ,			index=__A     ,			init_retrieval=__A     ,			)
      __UpperCAmelCase					=   True



    def 					__lowerCamelCase						(  self					):
     self.retriever.index.init_index()




    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     __UpperCAmelCase					,    __UpperCAmelCase					=   self.retriever._main_retrieve(__A     ,			__A					)
     return doc_ids, retrieved_doc_embeds






class UpperCAmelCase							(		UpperCAmelCase_							):



    def __init__(  self     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A=None					):
     if index is not None and index.is_initialized() and len(__A					) > 0:
      raise ValueError(
          'When using Ray for distributed fine-tuning, '
          'you\'ll need to provide the paths instead, '
          'as the dataset and the index are loaded '
          'separately. More info in examples/rag/use_own_knowledge_dataset.py '					)
     super().__init__(
         __A     ,			question_encoder_tokenizer=__A     ,			generator_tokenizer=__A     ,			index=__A     ,			init_retrieval=__A     ,			)
     __UpperCAmelCase					=   retrieval_workers
     if len(self.retrieval_workers					) > 0:
      ray.get(
          [
              worker.create_rag_retriever.remote(__A     ,			__A     ,			__A     ,			__A					)
              for worker in self.retrieval_workers
          ]					)



    def 					__lowerCamelCase						(  self					):
     logger.info('initializing retrieval'					)

     if len(self.retrieval_workers					) > 0:
      ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]					)
     else:
      # Non-distributed training. Load index into this same process.
      self.index.init_index()



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A					):
     if len(self.retrieval_workers					) > 0:
      # Select a random retrieval actor.
      __UpperCAmelCase					=   self.retrieval_workers[random.randint(0     ,			len(self.retrieval_workers					) - 1					)]
      __UpperCAmelCase					,    __UpperCAmelCase					=   ray.get(random_worker.retrieve.remote(__A     ,			__A					)					)
     else:
      __UpperCAmelCase					,    __UpperCAmelCase					=   self._main_retrieve(__A     ,			__A					)
     return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(__A					)



    @classmethod
    def 					__lowerCamelCase						(  cls     ,			__A     ,			__A=None     ,			**__A					):
     return super(__A     ,			cls					).get_tokenizers(__A     ,			__A     ,			**__A					)




    @classmethod
    def 					__lowerCamelCase						(  cls     ,			__A     ,			__A     ,			__A=None     ,			**__A					):
     __UpperCAmelCase					=   kwargs.pop('config'     ,			__A					) or RagConfig.from_pretrained(__A     ,			**__A					)
     __UpperCAmelCase					=   RagTokenizer.from_pretrained(__A     ,			config=__A					)
     __UpperCAmelCase					=   rag_tokenizer.question_encoder
     __UpperCAmelCase					=   rag_tokenizer.generator
     if indexed_dataset is not None:
      __UpperCAmelCase					=   'custom'
      __UpperCAmelCase					=   CustomHFIndex(config.retrieval_vector_size     ,			__A					)
     else:
      __UpperCAmelCase					=   cls._build_index(__A					)
     return cls(
         __A     ,			question_encoder_tokenizer=__A     ,			generator_tokenizer=__A     ,			retrieval_workers=__A     ,			index=__A     ,			)





