


"""simple docstring"""





import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer



class 		snake_case_(       a__			):






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           lowerCAmelCase					:	List[Any]       =   tempfile.mkdtemp()
           lowerCAmelCase					:	int       =   5

           # Realm tok
           lowerCAmelCase					:	Union[str, Any]       =   [
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''test''',
               '''question''',
               '''this''',
               '''is''',
               '''the''',
               '''first''',
               '''second''',
               '''third''',
               '''fourth''',
               '''fifth''',
               '''record''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           lowerCAmelCase					:	Any       =   os.path.join(self.tmpdirname					,				'''realm_tokenizer'''				)
           os.makedirs(UpperCamelCase_					,				exist_ok=UpperCamelCase_				)
           lowerCAmelCase					:	Tuple       =   os.path.join(UpperCamelCase_					,				VOCAB_FILES_NAMES['''vocab_file''']				)
           with open(self.vocab_file					,				'''w'''					,				encoding='''utf-8'''				) as vocab_writer:
                 vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)

           lowerCAmelCase					:	Optional[Any]       =   os.path.join(self.tmpdirname					,				'''realm_block_records'''				)
           os.makedirs(UpperCamelCase_					,				exist_ok=UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname					,				'''realm_tokenizer'''				)				)






     def      lowerCamelCase__     (	self  :					Union[str, Any]				):
           shutil.rmtree(self.tmpdirname				)






     def      lowerCamelCase__     (	self  :					int				):
           lowerCAmelCase					:	Tuple       =   RealmConfig(num_block_records=self.num_block_records				)
           return config






     def      lowerCamelCase__     (	self  :					int				):
           lowerCAmelCase					:	Dict       =   Dataset.from_dict(
               {
                   '''id''': ['''0''', '''1'''],
                   '''question''': ['''foo''', '''bar'''],
                   '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
               }				)
           return dataset






     def      lowerCamelCase__     (	self  :					int				):
           lowerCAmelCase					:	Optional[int]       =   np.array(
               [
                   b'''This is the first record''',
                   b'''This is the second record''',
                   b'''This is the third record''',
                   b'''This is the fourth record''',
                   b'''This is the fifth record''',
                   b'''This is a longer longer longer record''',
               ]					,				dtype=UpperCamelCase_					,				)
           return block_records






     def      lowerCamelCase__     (	self  :					List[str]				):
           lowerCAmelCase					:	str       =   RealmRetriever(
               block_records=self.get_dummy_block_records()					,				tokenizer=self.get_tokenizer()					,				)
           return retriever






     def      lowerCamelCase__     (	self  :					List[Any]				):
           lowerCAmelCase					:	int       =   self.get_config()
           lowerCAmelCase					:	Tuple       =   self.get_dummy_retriever()
           lowerCAmelCase					:	int       =   retriever.tokenizer

           lowerCAmelCase					:	Any       =   np.array([0, 3]					,				dtype='''long'''				)
           lowerCAmelCase					:	Optional[Any]       =   tokenizer(['''Test question''']				).input_ids
           lowerCAmelCase					:	List[Any]       =   tokenizer(
               ['''the fourth''']					,				add_special_tokens=UpperCamelCase_					,				return_token_type_ids=UpperCamelCase_					,				return_attention_mask=UpperCamelCase_					,				).input_ids
           lowerCAmelCase					:	Tuple       =   config.reader_seq_len

           lowerCAmelCase,      lowerCAmelCase,      lowerCAmelCase,      lowerCAmelCase					:	Dict       =   retriever(
               UpperCamelCase_					,				UpperCamelCase_					,				answer_ids=UpperCamelCase_					,				max_length=UpperCamelCase_					,				return_tensors='''np'''				)

           self.assertEqual(len(UpperCamelCase_				)					,				2				)
           self.assertEqual(len(UpperCamelCase_				)					,				2				)
           self.assertEqual(len(UpperCamelCase_				)					,				2				)
           self.assertEqual(concat_inputs.input_ids.shape					,				(2, 1_0)				)
           self.assertEqual(concat_inputs.attention_mask.shape					,				(2, 1_0)				)
           self.assertEqual(concat_inputs.token_type_ids.shape					,				(2, 1_0)				)
           self.assertEqual(concat_inputs.special_tokens_mask.shape					,				(2, 1_0)				)
           self.assertEqual(
               tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]				)					,				['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']					,				)
           self.assertEqual(
               tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]				)					,				['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']					,				)






     def      lowerCamelCase__     (	self  :					List[Any]				):
           lowerCAmelCase					:	List[Any]       =   self.get_config()
           lowerCAmelCase					:	Optional[int]       =   self.get_dummy_retriever()
           lowerCAmelCase					:	Dict       =   retriever.tokenizer

           lowerCAmelCase					:	str       =   np.array([0, 3, 5]					,				dtype='''long'''				)
           lowerCAmelCase					:	List[str]       =   tokenizer(['''Test question''']				).input_ids
           lowerCAmelCase					:	int       =   tokenizer(
               ['''the fourth''', '''longer longer''']					,				add_special_tokens=UpperCamelCase_					,				return_token_type_ids=UpperCamelCase_					,				return_attention_mask=UpperCamelCase_					,				).input_ids
           lowerCAmelCase					:	Union[str, Any]       =   config.reader_seq_len

           lowerCAmelCase,      lowerCAmelCase,      lowerCAmelCase,      lowerCAmelCase					:	Dict       =   retriever(
               UpperCamelCase_					,				UpperCamelCase_					,				answer_ids=UpperCamelCase_					,				max_length=UpperCamelCase_					,				return_tensors='''np'''				)

           self.assertEqual([False, True, True]					,				UpperCamelCase_				)
           self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]					,				UpperCamelCase_				)
           self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]					,				UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					List[str]				):
           lowerCAmelCase					:	Optional[int]       =   self.get_dummy_retriever()
           retriever.save_pretrained(os.path.join(self.tmpdirname					,				'''realm_block_records'''				)				)

           # Test local path
           lowerCAmelCase					:	Any       =   retriever.from_pretrained(os.path.join(self.tmpdirname					,				'''realm_block_records'''				)				)
           self.assertEqual(retriever.block_records[0]					,				b'''This is the first record'''				)

           # Test mocked remote path
           with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''				) as mock_hf_hub_download:
                 lowerCAmelCase					:	Optional[int]       =   os.path.join(
                     os.path.join(self.tmpdirname					,				'''realm_block_records'''				)					,				_REALM_BLOCK_RECORDS_FILENAME				)
                 lowerCAmelCase					:	Union[str, Any]       =   RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''				)

           self.assertEqual(retriever.block_records[0]					,				b'''This is the first record'''				)






"""simple docstring"""





# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool



class 		snake_case_(       a__			):
     __UpperCamelCase  =					'''philschmid/bart-large-cnn-samsum'''
     __UpperCamelCase  =					(
         '''This is a tool that summarizes an English text. It takes an input `text` containing the text to summarize, '''
         '''and returns a summary of the text.'''
     )
     __UpperCamelCase  =					'''summarizer'''
     __UpperCamelCase  =					AutoTokenizer
     __UpperCamelCase  =					AutoModelForSeqaSeqLM

     __UpperCamelCase  =					['''text''']
     __UpperCamelCase  =					['''text''']






     def      lowerCamelCase__     (	self  :					Dict					,				UpperCamelCase_  :					int				):
           return self.pre_processor(UpperCamelCase_					,				return_tensors='''pt'''					,				truncation=UpperCamelCase_				)






     def      lowerCamelCase__     (	self  :					Optional[int]					,				UpperCamelCase_  :					str				):
           return self.model.generate(**UpperCamelCase_				)[0]






     def      lowerCamelCase__     (	self  :					Any					,				UpperCamelCase_  :					Tuple				):
           return self.pre_processor.decode(UpperCamelCase_					,				skip_special_tokens=UpperCamelCase_					,				clean_up_tokenization_spaces=UpperCamelCase_				)


