



import json
import os
import pickle
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers import is_faiss_available
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bart.tokenization_bart import BartTokenizer
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.dpr.tokenization_dpr import DPRContextEncoderTokenizer, DPRQuestionEncoderTokenizer
from transformers.models.rag.configuration_rag import RagConfig
from transformers.models.rag.retrieval_rag import CustomHFIndex, RagRetriever
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_sentencepiece, require_tokenizers, require_torch


if is_faiss_available():
  import faiss



@require_faiss
class  __lowerCAmelCase     (	lowerCAmelCase__   ):




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    tempfile.mkdtemp()
    __lowerCamelCase													=    8

    # DPR tok
    __lowerCamelCase													=    [
        '''[UNK]''',
        '''[CLS]''',
        '''[SEP]''',
        '''[PAD]''',
        '''[MASK]''',
        '''want''',
        '''##want''',
        '''##ed''',
        '''wa''',
        '''un''',
        '''runn''',
        '''##ing''',
        ''',''',
        '''low''',
        '''lowest''',
    ]
    __lowerCamelCase													=    os.path.join(self.tmpdirname	,				'''dpr_tokenizer'''				)
    os.makedirs(__UpperCAmelCase	,				exist_ok=__UpperCAmelCase				)
    __lowerCamelCase													=    os.path.join(__UpperCAmelCase	,				DPR_VOCAB_FILES_NAMES['''vocab_file''']				)
    with open(self.vocab_file	,				'''w'''	,				encoding='''utf-8'''				) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)

    # BART tok
    __lowerCamelCase													=    [
        '''l''',
        '''o''',
        '''w''',
        '''e''',
        '''r''',
        '''s''',
        '''t''',
        '''i''',
        '''d''',
        '''n''',
        '''\u0120''',
        '''\u0120l''',
        '''\u0120n''',
        '''\u0120lo''',
        '''\u0120low''',
        '''er''',
        '''\u0120lowest''',
        '''\u0120newer''',
        '''\u0120wider''',
        '''<unk>''',
    ]
    __lowerCamelCase													=    dict(zip(__UpperCAmelCase	,				range(len(__UpperCAmelCase				)				)				)				)
    __lowerCamelCase													=    ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
    __lowerCamelCase													=    {'''unk_token''': '''<unk>'''}

    __lowerCamelCase													=    os.path.join(self.tmpdirname	,				'''bart_tokenizer'''				)
    os.makedirs(__UpperCAmelCase	,				exist_ok=__UpperCAmelCase				)
    __lowerCamelCase													=    os.path.join(__UpperCAmelCase	,				BART_VOCAB_FILES_NAMES['''vocab_file''']				)
    __lowerCamelCase													=    os.path.join(__UpperCAmelCase	,				BART_VOCAB_FILES_NAMES['''merges_file''']				)
    with open(self.vocab_file	,				'''w'''	,				encoding='''utf-8'''				) as fp:
       fp.write(json.dumps(__UpperCAmelCase				) + '''\n'''				)
    with open(self.merges_file	,				'''w'''	,				encoding='''utf-8'''				) as fp:
       fp.write('''\n'''.join(__UpperCAmelCase				)				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname	,				'''dpr_tokenizer'''				)				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return DPRContextEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname	,				'''dpr_tokenizer'''				)				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname	,				'''bart_tokenizer'''				)				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    shutil.rmtree(self.tmpdirname				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    Dataset.from_dict(
        {
            '''id''': ['''0''', '''1'''],
            '''text''': ['''foo''', '''bar'''],
            '''title''': ['''Foo''', '''Bar'''],
            '''embeddings''': [np.ones(self.retrieval_vector_size				), 2 * np.ones(self.retrieval_vector_size				)],
        }				)
    dataset.add_faiss_index('''embeddings'''	,				string_factory='''Flat'''	,				metric_type=faiss.METRIC_INNER_PRODUCT				)
    return dataset




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_dummy_dataset()
    __lowerCamelCase													=    RagConfig(
        retrieval_vector_size=self.retrieval_vector_size	,				question_encoder=DPRConfig().to_dict()	,				generator=BartConfig().to_dict()	,				)
    with patch('''transformers.models.rag.retrieval_rag.load_dataset'''				) as mock_load_dataset:
       __lowerCamelCase													=    dataset
       __lowerCamelCase													=    RagRetriever(
           __UpperCAmelCase	,				question_encoder_tokenizer=self.get_dpr_tokenizer()	,				generator_tokenizer=self.get_bart_tokenizer()	,				)
    return retriever




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_dummy_dataset()
    __lowerCamelCase													=    RagConfig(
        retrieval_vector_size=self.retrieval_vector_size	,				question_encoder=DPRConfig().to_dict()	,				generator=BartConfig().to_dict()	,				index_name='''custom'''	,				)
    if from_disk:
       __lowerCamelCase													=    os.path.join(self.tmpdirname	,				'''dataset'''				)
       __lowerCamelCase													=    os.path.join(self.tmpdirname	,				'''index.faiss'''				)
       dataset.get_index('''embeddings'''				).save(os.path.join(self.tmpdirname	,				'''index.faiss'''				)				)
       dataset.drop_index('''embeddings'''				)
       dataset.save_to_disk(os.path.join(self.tmpdirname	,				'''dataset'''				)				)
       del dataset
       __lowerCamelCase													=    RagRetriever(
           __UpperCAmelCase	,				question_encoder_tokenizer=self.get_dpr_tokenizer()	,				generator_tokenizer=self.get_bart_tokenizer()	,				)
    else:
       __lowerCamelCase													=    RagRetriever(
           __UpperCAmelCase	,				question_encoder_tokenizer=self.get_dpr_tokenizer()	,				generator_tokenizer=self.get_bart_tokenizer()	,				index=CustomHFIndex(config.retrieval_vector_size	,				__UpperCAmelCase				)	,				)
    return retriever




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    Dataset.from_dict(
        {
            '''id''': ['''0''', '''1'''],
            '''text''': ['''foo''', '''bar'''],
            '''title''': ['''Foo''', '''Bar'''],
            '''embeddings''': [np.ones(self.retrieval_vector_size + 1				), 2 * np.ones(self.retrieval_vector_size + 1				)],
        }				)
    dataset.add_faiss_index('''embeddings'''	,				string_factory='''Flat'''	,				metric_type=faiss.METRIC_INNER_PRODUCT				)

    __lowerCamelCase													=    os.path.join(self.tmpdirname	,				'''hf_bert_base.hnswSQ8_correct_phi_128.c_index'''				)
    dataset.save_faiss_index('''embeddings'''	,				index_file_name + '''.index.dpr'''				)
    pickle.dump(dataset['''id''']	,				open(index_file_name + '''.index_meta.dpr'''	,				'''wb'''				)				)

    __lowerCamelCase													=    os.path.join(self.tmpdirname	,				'''psgs_w100.tsv.pkl'''				)
    __lowerCamelCase													=    {sample['''id''']: [sample['''text'''], sample['''title''']] for sample in dataset}
    pickle.dump(__UpperCAmelCase	,				open(__UpperCAmelCase	,				'''wb'''				)				)

    __lowerCamelCase													=    RagConfig(
        retrieval_vector_size=self.retrieval_vector_size	,				question_encoder=DPRConfig().to_dict()	,				generator=BartConfig().to_dict()	,				index_name='''legacy'''	,				index_path=self.tmpdirname	,				)
    __lowerCamelCase													=    RagRetriever(
        __UpperCAmelCase	,				question_encoder_tokenizer=self.get_dpr_tokenizer()	,				generator_tokenizer=self.get_bart_tokenizer()				)
    return retriever




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    1
    __lowerCamelCase													=    self.get_dummy_canonical_hf_index_retriever()
    __lowerCamelCase													=    np.array(
        [np.ones(self.retrieval_vector_size				), -np.ones(self.retrieval_vector_size				)]	,				dtype=np.floataa				)
    __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    retriever.retrieve(__UpperCAmelCase	,				n_docs=__UpperCAmelCase				)
    self.assertEqual(retrieved_doc_embeds.shape	,				(2, n_docs, self.retrieval_vector_size)				)
    self.assertEqual(len(__UpperCAmelCase				)	,				2				)
    self.assertEqual(sorted(doc_dicts[0]				)	,				['''embeddings''', '''id''', '''text''', '''title''']				)
    self.assertEqual(len(doc_dicts[0]['''id''']				)	,				__UpperCAmelCase				)
    self.assertEqual(doc_dicts[0]['''id'''][0]	,				'''1'''				)  # max inner product is reached with second doc
    self.assertEqual(doc_dicts[1]['''id'''][0]	,				'''0'''				)  # max inner product is reached with first doc
    self.assertListEqual(doc_ids.tolist()	,				[[1], [0]]				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_dummy_canonical_hf_index_retriever()
    with tempfile.TemporaryDirectory() as tmp_dirname:
       with patch('''transformers.models.rag.retrieval_rag.load_dataset'''				) as mock_load_dataset:
          __lowerCamelCase													=    self.get_dummy_dataset()
          retriever.save_pretrained(__UpperCAmelCase				)
          __lowerCamelCase													=    RagRetriever.from_pretrained(__UpperCAmelCase				)
          self.assertIsInstance(__UpperCAmelCase	,				__UpperCAmelCase				)
          __lowerCamelCase													=    np.array(
              [np.ones(self.retrieval_vector_size				), -np.ones(self.retrieval_vector_size				)]	,				dtype=np.floataa				)
          __lowerCamelCase													=    retriever.retrieve(__UpperCAmelCase	,				n_docs=1				)
          self.assertTrue(out is not None				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    1
    __lowerCamelCase													=    self.get_dummy_custom_hf_index_retriever(from_disk=__UpperCAmelCase				)
    __lowerCamelCase													=    np.array(
        [np.ones(self.retrieval_vector_size				), -np.ones(self.retrieval_vector_size				)]	,				dtype=np.floataa				)
    __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    retriever.retrieve(__UpperCAmelCase	,				n_docs=__UpperCAmelCase				)
    self.assertEqual(retrieved_doc_embeds.shape	,				(2, n_docs, self.retrieval_vector_size)				)
    self.assertEqual(len(__UpperCAmelCase				)	,				2				)
    self.assertEqual(sorted(doc_dicts[0]				)	,				['''embeddings''', '''id''', '''text''', '''title''']				)
    self.assertEqual(len(doc_dicts[0]['''id''']				)	,				__UpperCAmelCase				)
    self.assertEqual(doc_dicts[0]['''id'''][0]	,				'''1'''				)  # max inner product is reached with second doc
    self.assertEqual(doc_dicts[1]['''id'''][0]	,				'''0'''				)  # max inner product is reached with first doc
    self.assertListEqual(doc_ids.tolist()	,				[[1], [0]]				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_dummy_custom_hf_index_retriever(from_disk=__UpperCAmelCase				)
    with tempfile.TemporaryDirectory() as tmp_dirname:
       retriever.save_pretrained(__UpperCAmelCase				)
       __lowerCamelCase													=    RagRetriever.from_pretrained(__UpperCAmelCase				)
       self.assertIsInstance(__UpperCAmelCase	,				__UpperCAmelCase				)
       __lowerCamelCase													=    np.array(
           [np.ones(self.retrieval_vector_size				), -np.ones(self.retrieval_vector_size				)]	,				dtype=np.floataa				)
       __lowerCamelCase													=    retriever.retrieve(__UpperCAmelCase	,				n_docs=1				)
       self.assertTrue(out is not None				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    1
    __lowerCamelCase													=    self.get_dummy_custom_hf_index_retriever(from_disk=__UpperCAmelCase				)
    __lowerCamelCase													=    np.array(
        [np.ones(self.retrieval_vector_size				), -np.ones(self.retrieval_vector_size				)]	,				dtype=np.floataa				)
    __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    retriever.retrieve(__UpperCAmelCase	,				n_docs=__UpperCAmelCase				)
    self.assertEqual(retrieved_doc_embeds.shape	,				(2, n_docs, self.retrieval_vector_size)				)
    self.assertEqual(len(__UpperCAmelCase				)	,				2				)
    self.assertEqual(sorted(doc_dicts[0]				)	,				['''embeddings''', '''id''', '''text''', '''title''']				)
    self.assertEqual(len(doc_dicts[0]['''id''']				)	,				__UpperCAmelCase				)
    self.assertEqual(doc_dicts[0]['''id'''][0]	,				'''1'''				)  # max inner product is reached with second doc
    self.assertEqual(doc_dicts[1]['''id'''][0]	,				'''0'''				)  # max inner product is reached with first doc
    self.assertListEqual(doc_ids.tolist()	,				[[1], [0]]				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_dummy_custom_hf_index_retriever(from_disk=__UpperCAmelCase				)
    with tempfile.TemporaryDirectory() as tmp_dirname:
       retriever.save_pretrained(__UpperCAmelCase				)
       __lowerCamelCase													=    RagRetriever.from_pretrained(__UpperCAmelCase				)
       self.assertIsInstance(__UpperCAmelCase	,				__UpperCAmelCase				)
       __lowerCamelCase													=    np.array(
           [np.ones(self.retrieval_vector_size				), -np.ones(self.retrieval_vector_size				)]	,				dtype=np.floataa				)
       __lowerCamelCase													=    retriever.retrieve(__UpperCAmelCase	,				n_docs=1				)
       self.assertTrue(out is not None				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    1
    __lowerCamelCase													=    self.get_dummy_legacy_index_retriever()
    __lowerCamelCase													=    np.array(
        [np.ones(self.retrieval_vector_size				), -np.ones(self.retrieval_vector_size				)]	,				dtype=np.floataa				)
    __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    retriever.retrieve(__UpperCAmelCase	,				n_docs=__UpperCAmelCase				)
    self.assertEqual(retrieved_doc_embeds.shape	,				(2, n_docs, self.retrieval_vector_size)				)
    self.assertEqual(len(__UpperCAmelCase				)	,				2				)
    self.assertEqual(sorted(doc_dicts[0]				)	,				['''text''', '''title''']				)
    self.assertEqual(len(doc_dicts[0]['''text''']				)	,				__UpperCAmelCase				)
    self.assertEqual(doc_dicts[0]['''text'''][0]	,				'''bar'''				)  # max inner product is reached with second doc
    self.assertEqual(doc_dicts[1]['''text'''][0]	,				'''foo'''				)  # max inner product is reached with first doc
    self.assertListEqual(doc_ids.tolist()	,				[[1], [0]]				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_dummy_legacy_index_retriever()
    with tempfile.TemporaryDirectory() as tmp_dirname:
       retriever.save_pretrained(__UpperCAmelCase				)
       __lowerCamelCase													=    RagRetriever.from_pretrained(__UpperCAmelCase				)
       self.assertIsInstance(__UpperCAmelCase	,				__UpperCAmelCase				)
       __lowerCamelCase													=    np.array(
           [np.ones(self.retrieval_vector_size				), -np.ones(self.retrieval_vector_size				)]	,				dtype=np.floataa				)
       __lowerCamelCase													=    retriever.retrieve(__UpperCAmelCase	,				n_docs=1				)
       self.assertTrue(out is not None				)




 @require_torch
 @require_tokenizers
 @require_sentencepiece
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    import torch

    __lowerCamelCase													=    1
    __lowerCamelCase													=    self.get_dummy_canonical_hf_index_retriever()
    __lowerCamelCase													=    [[5, 7], [10, 11]]
    __lowerCamelCase													=    np.array(
        [np.ones(self.retrieval_vector_size				), -np.ones(self.retrieval_vector_size				)]	,				dtype=np.floataa				)
    __lowerCamelCase													=    retriever(__UpperCAmelCase	,				__UpperCAmelCase	,				prefix=retriever.config.generator.prefix	,				n_docs=__UpperCAmelCase				)
    __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    (
        out['''context_input_ids'''],
        out['''context_attention_mask'''],
        out['''retrieved_doc_embeds'''],
    )
    self.assertEqual(retrieved_doc_embeds.shape	,				(2, n_docs, self.retrieval_vector_size)				)
    self.assertIsInstance(__UpperCAmelCase	,				__UpperCAmelCase				)
    self.assertIsInstance(__UpperCAmelCase	,				__UpperCAmelCase				)
    self.assertIsInstance(__UpperCAmelCase	,				np.ndarray				)

    __lowerCamelCase													=    retriever(
        __UpperCAmelCase	,				__UpperCAmelCase	,				prefix=retriever.config.generator.prefix	,				n_docs=__UpperCAmelCase	,				return_tensors='''pt'''	,				)
    __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    (  # noqa: F841
        out['''context_input_ids'''],
        out['''context_attention_mask'''],
        out['''retrieved_doc_embeds'''],
        out['''doc_ids'''],
    )
    self.assertEqual(retrieved_doc_embeds.shape	,				(2, n_docs, self.retrieval_vector_size)				)
    self.assertIsInstance(__UpperCAmelCase	,				torch.Tensor				)
    self.assertIsInstance(__UpperCAmelCase	,				torch.Tensor				)
    self.assertIsInstance(__UpperCAmelCase	,				torch.Tensor				)




 @require_torch
 @require_tokenizers
 @require_sentencepiece
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_dpr_ctx_encoder_tokenizer()
    __lowerCamelCase													=    1
    __lowerCamelCase													=    self.get_dummy_custom_hf_index_retriever(from_disk=__UpperCAmelCase				)
    retriever.set_ctx_encoder_tokenizer(__UpperCAmelCase				)

    __lowerCamelCase													=    [[5, 7], [10, 11]]
    __lowerCamelCase													=    np.array(
        [np.ones(self.retrieval_vector_size				), -np.ones(self.retrieval_vector_size				)]	,				dtype=np.floataa				)
    __lowerCamelCase													=    retriever(__UpperCAmelCase	,				__UpperCAmelCase	,				prefix=retriever.config.generator.prefix	,				n_docs=__UpperCAmelCase				)

    self.assertEqual(
        len(__UpperCAmelCase				)	,				6				)  # check whether the retriever output consist of 6 attributes including tokenized docs
    self.assertEqual(
        all(k in out for k in ('''tokenized_doc_ids''', '''tokenized_doc_attention_mask''')				)	,				__UpperCAmelCase				)  # check for doc token related keys in dictionary.











import unittest

from transformers import RoFormerTokenizer, RoFormerTokenizerFast
from transformers.testing_utils import require_rjieba, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_rjieba
@require_tokenizers
class  __lowerCAmelCase     (	lowerCAmelCase__  ,	unittest.TestCase   ):
 lowerCAmelCase__       					=  RoFormerTokenizer
 lowerCAmelCase__       					=  RoFormerTokenizerFast
 lowerCAmelCase__       					=  True
 lowerCAmelCase__       					=  True




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    super().setUp()




 def        lowerCamelCase      (						self	,				**__UpperCAmelCase				):



    '''simple docstring'''
    return self.tokenizer_class.from_pretrained('''junnyu/roformer_chinese_base'''	,				**__UpperCAmelCase				)




 def        lowerCamelCase      (						self	,				**__UpperCAmelCase				):



    '''simple docstring'''
    return self.rust_tokenizer_class.from_pretrained('''junnyu/roformer_chinese_base'''	,				**__UpperCAmelCase				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''永和服装饰品有限公司,今天天气非常好'''
    __lowerCamelCase													=    '''永和 服装 饰品 有限公司 , 今 天 天 气 非常 好'''
    return input_text, output_text




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_tokenizer()
    __lowerCamelCase							,__lowerCamelCase													=    self.get_chinese_input_output_texts()
    __lowerCamelCase													=    tokenizer.tokenize(__UpperCAmelCase				)

    self.assertListEqual(__UpperCAmelCase	,				output_text.split()				)

    __lowerCamelCase													=    tokens + [tokenizer.unk_token]
    __lowerCamelCase													=    [22943, 21332, 34431, 45904, 117, 306, 1231, 1231, 2653, 33994, 1266, 100]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)	,				__UpperCAmelCase				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.get_rust_tokenizer()
    __lowerCamelCase							,__lowerCamelCase													=    self.get_chinese_input_output_texts()
    __lowerCamelCase													=    tokenizer.tokenize(__UpperCAmelCase				)
    self.assertListEqual(__UpperCAmelCase	,				output_text.split()				)
    __lowerCamelCase													=    tokens + [tokenizer.unk_token]
    __lowerCamelCase													=    [22943, 21332, 34431, 45904, 117, 306, 1231, 1231, 2653, 33994, 1266, 100]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)	,				__UpperCAmelCase				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    pass




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    pass




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    pass






