


import logging
import os
from typing import List, Tuple

import numpy as np
import psutil
import torch
import torch.distributed as dist

from transformers import RagRetriever


snake_case							: List[str]      =   logging.getLogger(__name__)







class  snake_case_     (lowerCamelCase_   ):
					def __init__(   self     :Optional[Any]    ,__snake_case     :int    ,__snake_case     :List[str]    ,__snake_case     :List[Any]    ,__snake_case     :Dict=None   )      ->					Union[str, Any]:
												super().__init__(
												    __snake_case    ,question_encoder_tokenizer=__snake_case    ,generator_tokenizer=__snake_case    ,index=__snake_case    ,init_retrieval=__snake_case    ,)
												a__    =  None
					def 				lowerCamelCase__(   self     :List[str]    ,__snake_case     :int   )      ->					Optional[Any]:
												logger.info('initializing retrieval'   )

												# initializing a separate process group for retrieval as the default
												# nccl backend doesn't support gather/scatter operations while gloo
												# is too slow to replace nccl for the core gpu communication
												if dist.is_initialized():
																			logger.info('dist initialized'   )
																			# needs to be set manually
																			a__    =  self._infer_socket_ifname()
																			# avoid clash with the NCCL port
																			a__    =  str(distributed_port + 1   )
																			a__    =  dist.new_group(ranks=__snake_case    ,backend='gloo'   )

												# initialize retriever only on the main worker
												if not dist.is_initialized() or self._is_main():
																			logger.info('dist not initialized / main'   )
																			self.index.init_index()

												# all processes wait untill the retriever is initialized by the main process
												if dist.is_initialized():
																			torch.distributed.barrier(group=self.process_group   )
					def 				lowerCamelCase__(   self     :Union[str, Any]   )      ->					Any:
												return dist.get_rank(group=self.process_group   ) == 0
					def 				lowerCamelCase__(   self     :List[Any]    ,__snake_case     :Tuple    ,__snake_case     :List[str]    ,__snake_case     :int=torch.floataa   )      ->					int:
												a__    =  torch.empty(__snake_case    ,dtype=__snake_case   )
												dist.scatter(__snake_case    ,src=0    ,scatter_list=__snake_case    ,group=self.process_group   )
												return target_tensor
					def 				lowerCamelCase__(   self     :Optional[int]   )      ->					Tuple:
												a__    =  psutil.net_if_addrs()
												# a hacky way to deal with varying network interface names
												a__    =  next((addr for addr in addrs if addr.startswith('e'   ))    ,__snake_case   )
												return ifname



					def 				lowerCamelCase__(   self     :int    ,__snake_case     :np.ndarray    ,__snake_case     :int   )      ->					Tuple[np.ndarray, List[dict]]:

												# single GPU training
												if not dist.is_initialized():
																			a__   ,      a__    =  self._main_retrieve(__snake_case    ,__snake_case   )
																			return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(__snake_case   )

												# distributed training
												a__    =  dist.get_world_size(group=self.process_group   )

												# gather logic
												a__    =  None
												if self._is_main():
																			a__    =  [torch.empty(question_hidden_states.shape    ,dtype=torch.floataa   ) for _ in range(__snake_case   )]
												dist.gather(torch.tensor(__snake_case   )    ,dst=0    ,gather_list=__snake_case    ,group=self.process_group   )

												# scatter logic
												a__    =  question_hidden_states.shape[0]
												a__    =  []
												a__    =  []
												if self._is_main():
																			assert len(__snake_case   ) == world_size
																			a__   ,      a__    =  self._main_retrieve(torch.cat(__snake_case   ).numpy()    ,__snake_case   )
																			a__   ,      a__    =  torch.tensor(__snake_case   ), torch.tensor(__snake_case   )
																			a__    =  self._chunk_tensor(__snake_case    ,__snake_case   )
																			a__    =  self._chunk_tensor(__snake_case    ,__snake_case   )
												a__    =  self._scattered(__snake_case    ,[n_queries, n_docs]    ,target_type=torch.intaa   )
												a__    =  self._scattered(__snake_case    ,[n_queries, n_docs, question_hidden_states.shape[1]]   )

												return retrieved_doc_embeds.numpy(), doc_ids.numpy(), self.index.get_doc_dicts(__snake_case   )








import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

snake_case							: Dict      =   logging.get_logger(__name__)


snake_case							: Any      =   {name: getattr(transformers, name + '''Fast''') for name in SLOW_TO_FAST_CONVERTERS}






def 							__lowercase     (				__lowerCAmelCase							:       Optional[Any]      ,					__lowerCAmelCase							:       Union[str, Any]      ,					__lowerCAmelCase							:       Any      ,					__lowerCAmelCase							:       Optional[Any]     ):
							if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
														raise ValueError(F'Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()     )}.'     )

							if tokenizer_name is None:
														a__    =  TOKENIZER_CLASSES
							else:
														a__    =  {tokenizer_name: getattr(__lowerCAmelCase      ,					tokenizer_name + 'Fast'     )}

							logger.info(F'Loading tokenizer classes: {tokenizer_names}'     )

							for tokenizer_name in tokenizer_names:
														a__    =  TOKENIZER_CLASSES[tokenizer_name]

														a__    =  True
														if checkpoint_name is None:
																					a__    =  list(tokenizer_class.max_model_input_sizes.keys()     )
														else:
																					a__    =  [checkpoint_name]

														logger.info(F'For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}'     )

														for checkpoint in checkpoint_names:
																					logger.info(F'Loading {tokenizer_class.__class__.__name__} {checkpoint}'     )

																					# Load tokenizer
																					a__    =  tokenizer_class.from_pretrained(__lowerCAmelCase      ,					force_download=__lowerCAmelCase     )

																					# Save fast tokenizer
																					logger.info(F'Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}'     )

																					# For organization names we create sub-directories
																					if "/" in checkpoint:
																												a__   ,      a__    =  checkpoint.split('/'     )
																												a__    =  os.path.join(__lowerCAmelCase      ,					__lowerCAmelCase     )
																					elif add_prefix:
																												a__    =  checkpoint
																												a__    =  dump_path
																					else:
																												a__    =  None
																												a__    =  dump_path

																					logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'     )

																					if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()     )[0]:
																												a__    =  list(tokenizer.pretrained_vocab_files_map.values()     )[0][checkpoint]
																												a__    =  file_path.split(__lowerCAmelCase     )[-1][0]
																												if next_char == "/":
																																			a__    =  os.path.join(__lowerCAmelCase      ,					__lowerCAmelCase     )
																																			a__    =  None

																												logger.info(F'=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}'     )

																					a__    =  tokenizer.save_pretrained(
																					    __lowerCAmelCase      ,					legacy_format=__lowerCAmelCase      ,					filename_prefix=__lowerCAmelCase     )
																					logger.info(F'=> File names {file_names}'     )

																					for file_name in file_names:
																												if not file_name.endswith('tokenizer.json'     ):
																																			os.remove(__lowerCAmelCase     )
																																			logger.info(F'=> removing {file_name}'     )


if __name__ == "__main__":
						snake_case							: Union[str, Any]      =   argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '''--dump_path''', default=None, type=str, required=True, help='''Path to output generated fast tokenizer files.'''
						)
						parser.add_argument(
						    '''--tokenizer_name''',
						    default=None,
						    type=str,
						    help=(
						        f"""Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will """
						        '''download and convert all the checkpoints from AWS.'''
						    ),
						)
						parser.add_argument(
						    '''--checkpoint_name''',
						    default=None,
						    type=str,
						    help='''Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.''',
						)
						parser.add_argument(
						    '''--force_download''',
						    action='''store_true''',
						    help='''Re-download checkpoints.''',
						)
						snake_case							: List[str]      =   parser.parse_args()

						convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)




