import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer



class  UpperCamelCase				( __lowercase ):

     '''simple docstring'''

     def 		UpperCamelCase_						(					self						) ->				List[Any]:






           """simple docstring"""
           _lowerCamelCase			    =				tempfile.mkdtemp()
           _lowerCamelCase			    =				5

           # Realm tok
           _lowerCamelCase			    =				[
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''test''',
               '''question''',
               '''this''',
               '''is''',
               '''the''',
               '''first''',
               '''second''',
               '''third''',
               '''fourth''',
               '''fifth''',
               '''record''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           _lowerCamelCase			    =				os.path.join(self.tmpdirname				,				'''realm_tokenizer'''						)
           os.makedirs(A_				,				exist_ok=A_						)
           _lowerCamelCase			    =				os.path.join(A_				,				VOCAB_FILES_NAMES['''vocab_file''']						)
           with open(self.vocab_file				,				'''w'''				,				encoding='''utf-8'''						) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]						)						)

           _lowerCamelCase			    =				os.path.join(self.tmpdirname				,				'''realm_block_records'''						)
           os.makedirs(A_				,				exist_ok=A_						)



     def 		UpperCamelCase_						(					self						) ->				RealmTokenizer:






           """simple docstring"""
           return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname				,				'''realm_tokenizer'''						)						)



     def 		UpperCamelCase_						(					self						) ->				int:






           """simple docstring"""
           shutil.rmtree(self.tmpdirname						)



     def 		UpperCamelCase_						(					self						) ->				Optional[Any]:






           """simple docstring"""
           _lowerCamelCase			    =				RealmConfig(num_block_records=self.num_block_records						)
           return config



     def 		UpperCamelCase_						(					self						) ->				str:






           """simple docstring"""
           _lowerCamelCase			    =				Dataset.from_dict(
               {
                   '''id''': ['''0''', '''1'''],
                   '''question''': ['''foo''', '''bar'''],
                   '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
               }						)
           return dataset



     def 		UpperCamelCase_						(					self						) ->				Optional[int]:






           """simple docstring"""
           _lowerCamelCase			    =				np.array(
               [
                   B'''This is the first record''',
                   B'''This is the second record''',
                   B'''This is the third record''',
                   B'''This is the fourth record''',
                   B'''This is the fifth record''',
                   B'''This is a longer longer longer record''',
               ]				,				dtype=A_				,				)
           return block_records



     def 		UpperCamelCase_						(					self						) ->				List[str]:






           """simple docstring"""
           _lowerCamelCase			    =				RealmRetriever(
               block_records=self.get_dummy_block_records()				,				tokenizer=self.get_tokenizer()				,				)
           return retriever



     def 		UpperCamelCase_						(					self						) ->				Optional[Any]:






           """simple docstring"""
           _lowerCamelCase			    =				self.get_config()
           _lowerCamelCase			    =				self.get_dummy_retriever()
           _lowerCamelCase			    =				retriever.tokenizer

           _lowerCamelCase			    =				np.array([0, 3]				,				dtype='''long'''						)
           _lowerCamelCase			    =				tokenizer(['''Test question''']						).input_ids
           _lowerCamelCase			    =				tokenizer(
               ['''the fourth''']				,				add_special_tokens=A_				,				return_token_type_ids=A_				,				return_attention_mask=A_				,				).input_ids
           _lowerCamelCase			    =				config.reader_seq_len

           _lowerCamelCase    ,					_lowerCamelCase    ,					_lowerCamelCase    ,					_lowerCamelCase			    =				retriever(
               A_				,				A_				,				answer_ids=A_				,				max_length=A_				,				return_tensors='''np'''						)

           self.assertEqual(len(A_						)				,				2						)
           self.assertEqual(len(A_						)				,				2						)
           self.assertEqual(len(A_						)				,				2						)
           self.assertEqual(concat_inputs.input_ids.shape				,				(2, 10)						)
           self.assertEqual(concat_inputs.attention_mask.shape				,				(2, 10)						)
           self.assertEqual(concat_inputs.token_type_ids.shape				,				(2, 10)						)
           self.assertEqual(concat_inputs.special_tokens_mask.shape				,				(2, 10)						)
           self.assertEqual(
               tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]						)				,				['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']				,				)
           self.assertEqual(
               tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]						)				,				['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']				,				)



     def 		UpperCamelCase_						(					self						) ->				List[Any]:






           """simple docstring"""
           _lowerCamelCase			    =				self.get_config()
           _lowerCamelCase			    =				self.get_dummy_retriever()
           _lowerCamelCase			    =				retriever.tokenizer

           _lowerCamelCase			    =				np.array([0, 3, 5]				,				dtype='''long'''						)
           _lowerCamelCase			    =				tokenizer(['''Test question''']						).input_ids
           _lowerCamelCase			    =				tokenizer(
               ['''the fourth''', '''longer longer''']				,				add_special_tokens=A_				,				return_token_type_ids=A_				,				return_attention_mask=A_				,				).input_ids
           _lowerCamelCase			    =				config.reader_seq_len

           _lowerCamelCase    ,					_lowerCamelCase    ,					_lowerCamelCase    ,					_lowerCamelCase			    =				retriever(
               A_				,				A_				,				answer_ids=A_				,				max_length=A_				,				return_tensors='''np'''						)

           self.assertEqual([False, True, True]				,				A_						)
           self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]				,				A_						)
           self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]				,				A_						)





     def 		UpperCamelCase_						(					self						) ->				List[str]:






           """simple docstring"""
           _lowerCamelCase			    =				self.get_dummy_retriever()
           retriever.save_pretrained(os.path.join(self.tmpdirname				,				'''realm_block_records'''						)						)

           # Test local path
           _lowerCamelCase			    =				retriever.from_pretrained(os.path.join(self.tmpdirname				,				'''realm_block_records'''						)						)
           self.assertEqual(retriever.block_records[0]				,				B'''This is the first record'''						)

           # Test mocked remote path
           with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''						) as mock_hf_hub_download:
                 _lowerCamelCase			    =				os.path.join(
                     os.path.join(self.tmpdirname				,				'''realm_block_records'''						)				,				_REALM_BLOCK_RECORDS_FILENAME						)
                 _lowerCamelCase			    =				RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''						)

           self.assertEqual(retriever.block_records[0]				,				B'''This is the first record'''						)
import os
import tempfile
import unittest

from transformers.models.marian.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
from transformers.testing_utils import slow
from transformers.utils import cached_property



@unittest.skipUnless(os.path.exists(__lowercase )   ,  'Tatoeba directory does not exist.' )
class  UpperCamelCase				( unittest.TestCase ):

     '''simple docstring'''

     @cached_property
     def 		UpperCamelCase_						(					self						) ->				int:






           """simple docstring"""
           _lowerCamelCase			    =				tempfile.mkdtemp()
           return TatoebaConverter(save_dir=A_						)



     @slow
     def 		UpperCamelCase_						(					self						) ->				Any:






           """simple docstring"""
           self.resolver.convert_models(['''heb-eng''']						)





     @slow
     def 		UpperCamelCase_						(					self						) ->				Tuple:






           """simple docstring"""
           _lowerCamelCase    ,					_lowerCamelCase			    =				self.resolver.write_model_card('''opus-mt-he-en'''				,				dry_run=A_						)
           assert mmeta["long_pair"] == "heb-eng"