



import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer





class 		UpperCAmelCase						(						UpperCAmelCase__				):





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  tempfile.mkdtemp()
         _lowerCAmelCase 			=  5

         # Realm tok
         _lowerCAmelCase 			=  [
             '''[UNK]''',
             '''[CLS]''',
             '''[SEP]''',
             '''[PAD]''',
             '''[MASK]''',
             '''test''',
             '''question''',
             '''this''',
             '''is''',
             '''the''',
             '''first''',
             '''second''',
             '''third''',
             '''fourth''',
             '''fifth''',
             '''record''',
             '''want''',
             '''##want''',
             '''##ed''',
             '''wa''',
             '''un''',
             '''runn''',
             '''##ing''',
             ''',''',
             '''low''',
             '''lowest''',
         ]
         _lowerCAmelCase 			=  os.path.join(self.tmpdirname       ,     '''realm_tokenizer'''			)
         os.makedirs(__lowerCAmelCase       ,     exist_ok=__lowerCAmelCase			)
         _lowerCAmelCase 			=  os.path.join(__lowerCAmelCase       ,     VOCAB_FILES_NAMES['''vocab_file''']			)
         with open(self.vocab_file       ,     '''w'''       ,     encoding='''utf-8'''			) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)

         _lowerCAmelCase 			=  os.path.join(self.tmpdirname       ,     '''realm_block_records'''			)
         os.makedirs(__lowerCAmelCase       ,     exist_ok=__lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname       ,     '''realm_tokenizer'''			)			)





   def 	__lowerCAmelCase							(	self			):
         shutil.rmtree(self.tmpdirname			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  RealmConfig(num_block_records=self.num_block_records			)
         return config





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  Dataset.from_dict(
             {
                 '''id''': ['''0''', '''1'''],
                 '''question''': ['''foo''', '''bar'''],
                 '''answers''': [['''Foo''', '''Bar'''], ['''Bar''']],
             }			)
         return dataset





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  np.array(
             [
                 b'''This is the first record''',
                 b'''This is the second record''',
                 b'''This is the third record''',
                 b'''This is the fourth record''',
                 b'''This is the fifth record''',
                 b'''This is a longer longer longer record''',
             ]       ,     dtype=__lowerCAmelCase       ,     )
         return block_records





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  RealmRetriever(
             block_records=self.get_dummy_block_records()       ,     tokenizer=self.get_tokenizer()       ,     )
         return retriever





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.get_config()
         _lowerCAmelCase 			=  self.get_dummy_retriever()
         _lowerCAmelCase 			=  retriever.tokenizer

         _lowerCAmelCase 			=  np.array([0, 3]       ,     dtype='''long'''			)
         _lowerCAmelCase 			=  tokenizer(['''Test question''']			).input_ids
         _lowerCAmelCase 			=  tokenizer(
             ['''the fourth''']       ,     add_special_tokens=__lowerCAmelCase       ,     return_token_type_ids=__lowerCAmelCase       ,     return_attention_mask=__lowerCAmelCase       ,     ).input_ids
         _lowerCAmelCase 			=  config.reader_seq_len

         _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase 			=  retriever(
             __lowerCAmelCase       ,     __lowerCAmelCase       ,     answer_ids=__lowerCAmelCase       ,     max_length=__lowerCAmelCase       ,     return_tensors='''np'''			)

         self.assertEqual(len(__lowerCAmelCase			)       ,     2			)
         self.assertEqual(len(__lowerCAmelCase			)       ,     2			)
         self.assertEqual(len(__lowerCAmelCase			)       ,     2			)
         self.assertEqual(concat_inputs.input_ids.shape       ,     (2, 10)			)
         self.assertEqual(concat_inputs.attention_mask.shape       ,     (2, 10)			)
         self.assertEqual(concat_inputs.token_type_ids.shape       ,     (2, 10)			)
         self.assertEqual(concat_inputs.special_tokens_mask.shape       ,     (2, 10)			)
         self.assertEqual(
             tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0]			)       ,     ['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''first''', '''record''', '''[SEP]''']       ,     )
         self.assertEqual(
             tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1]			)       ,     ['''[CLS]''', '''test''', '''question''', '''[SEP]''', '''this''', '''is''', '''the''', '''fourth''', '''record''', '''[SEP]''']       ,     )





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.get_config()
         _lowerCAmelCase 			=  self.get_dummy_retriever()
         _lowerCAmelCase 			=  retriever.tokenizer

         _lowerCAmelCase 			=  np.array([0, 3, 5]       ,     dtype='''long'''			)
         _lowerCAmelCase 			=  tokenizer(['''Test question''']			).input_ids
         _lowerCAmelCase 			=  tokenizer(
             ['''the fourth''', '''longer longer''']       ,     add_special_tokens=__lowerCAmelCase       ,     return_token_type_ids=__lowerCAmelCase       ,     return_attention_mask=__lowerCAmelCase       ,     ).input_ids
         _lowerCAmelCase 			=  config.reader_seq_len

         _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase			,   _lowerCAmelCase 			=  retriever(
             __lowerCAmelCase       ,     __lowerCAmelCase       ,     answer_ids=__lowerCAmelCase       ,     max_length=__lowerCAmelCase       ,     return_tensors='''np'''			)

         self.assertEqual([False, True, True]       ,     __lowerCAmelCase			)
         self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]       ,     __lowerCAmelCase			)
         self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]       ,     __lowerCAmelCase			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  self.get_dummy_retriever()
         retriever.save_pretrained(os.path.join(self.tmpdirname       ,     '''realm_block_records'''			)			)

         # Test local path
         _lowerCAmelCase 			=  retriever.from_pretrained(os.path.join(self.tmpdirname       ,     '''realm_block_records'''			)			)
         self.assertEqual(retriever.block_records[0]       ,     b'''This is the first record'''			)

         # Test mocked remote path
         with patch('''transformers.models.realm.retrieval_realm.hf_hub_download'''			) as mock_hf_hub_download:
               _lowerCAmelCase 			=  os.path.join(
                   os.path.join(self.tmpdirname       ,     '''realm_block_records'''			)       ,     _REALM_BLOCK_RECORDS_FILENAME			)
               _lowerCAmelCase 			=  RealmRetriever.from_pretrained('''google/realm-cc-news-pretrained-openqa'''			)

         self.assertEqual(retriever.block_records[0]       ,     b'''This is the first record'''			)




import numpy





class 		UpperCAmelCase						:





   def __init__(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  input_array

         # Random initial weights are assigned where first argument is the
         # number of nodes in previous layer and second argument is the
         # number of nodes in the next layer.

         # Random initial weights are assigned.
         # self.input_array.shape[1] is used to represent number of nodes in input layer.
         # First hidden layer consists of 4 nodes.
         _lowerCAmelCase 			=  numpy.random.rand(
             self.input_array.shape[1]       ,     4			)

         # Random initial values for the first hidden layer.
         # First hidden layer has 4 nodes.
         # Second hidden layer has 3 nodes.
         _lowerCAmelCase 			=  numpy.random.rand(
             4       ,     3			)

         # Random initial values for the second hidden layer.
         # Second hidden layer has 3 nodes.
         # Output layer has 1 node.
         _lowerCAmelCase 			=  numpy.random.rand(3       ,     1			)

         # Real output values provided.
         _lowerCAmelCase 			=  output_array

         # Predicted output values by the neural network.
         # Predicted_output array initially consists of zeroes.
         _lowerCAmelCase 			=  numpy.zeros(output_array.shape			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  sigmoid(
             numpy.dot(self.input_array       ,     self.input_layer_and_first_hidden_layer_weights			)			)

         # layer_between_first_hidden_layer_and_second_hidden_layer is the layer
         # connecting the first hidden set of nodes with the second hidden set of nodes.
         _lowerCAmelCase 			=  sigmoid(
             numpy.dot(
                 self.layer_between_input_and_first_hidden_layer       ,     self.first_hidden_layer_and_second_hidden_layer_weights       ,     )			)

         # layer_between_second_hidden_layer_and_output is the layer connecting
         # second hidden layer with the output node.
         _lowerCAmelCase 			=  sigmoid(
             numpy.dot(
                 self.layer_between_first_hidden_layer_and_second_hidden_layer       ,     self.second_hidden_layer_and_output_layer_weights       ,     )			)

         return self.layer_between_second_hidden_layer_and_output





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  numpy.dot(
             self.layer_between_first_hidden_layer_and_second_hidden_layer.T       ,     2
             * (self.output_array - self.predicted_output)
             * sigmoid_derivative(self.predicted_output			)       ,     )
         _lowerCAmelCase 			=  numpy.dot(
             self.layer_between_input_and_first_hidden_layer.T       ,     numpy.dot(
                 2
                 * (self.output_array - self.predicted_output)
                 * sigmoid_derivative(self.predicted_output			)       ,     self.second_hidden_layer_and_output_layer_weights.T       ,     )
             * sigmoid_derivative(
                 self.layer_between_first_hidden_layer_and_second_hidden_layer			)       ,     )
         _lowerCAmelCase 			=  numpy.dot(
             self.input_array.T       ,     numpy.dot(
                 numpy.dot(
                     2
                     * (self.output_array - self.predicted_output)
                     * sigmoid_derivative(self.predicted_output			)       ,     self.second_hidden_layer_and_output_layer_weights.T       ,     )
                 * sigmoid_derivative(
                     self.layer_between_first_hidden_layer_and_second_hidden_layer			)       ,     self.first_hidden_layer_and_second_hidden_layer_weights.T       ,     )
             * sigmoid_derivative(self.layer_between_input_and_first_hidden_layer			)       ,     )

         self.input_layer_and_first_hidden_layer_weights += (
             updated_input_layer_and_first_hidden_layer_weights
         )
         self.first_hidden_layer_and_second_hidden_layer_weights += (
             updated_first_hidden_layer_and_second_hidden_layer_weights
         )
         self.second_hidden_layer_and_output_layer_weights += (
             updated_second_hidden_layer_and_output_layer_weights
         )





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase       ,     _lowerCAmelCase       ,     _lowerCAmelCase			):
         for iteration in range(1       ,     iterations + 1			):
               _lowerCAmelCase 			=  self.feedforward()
               self.back_propagation()
               if give_loss:
                     _lowerCAmelCase 			=  numpy.mean(numpy.square(output - self.feedforward()			)			)
                     print(F'''Iteration {iteration} Loss: {loss}'''			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  input_arr

         _lowerCAmelCase 			=  sigmoid(
             numpy.dot(self.array       ,     self.input_layer_and_first_hidden_layer_weights			)			)

         _lowerCAmelCase 			=  sigmoid(
             numpy.dot(
                 self.layer_between_input_and_first_hidden_layer       ,     self.first_hidden_layer_and_second_hidden_layer_weights       ,     )			)

         _lowerCAmelCase 			=  sigmoid(
             numpy.dot(
                 self.layer_between_first_hidden_layer_and_second_hidden_layer       ,     self.second_hidden_layer_and_output_layer_weights       ,     )			)

         return int(self.layer_between_second_hidden_layer_and_output > 0.6			)






def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	numpy.ndarray				)->numpy.ndarray:
      return 1 / (1 + numpy.exp(-value				))
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	numpy.ndarray				)->numpy.ndarray:
      return (value) * (1 - (value))
def 							UpperCAmelCase__		(				)->int:
      _lowerCAmelCase 			=  numpy.array(
          (
              [0, 0, 0],
              [0, 0, 1],
              [0, 1, 0],
              [0, 1, 1],
              [1, 0, 0],
              [1, 0, 1],
              [1, 1, 0],
              [1, 1, 1],
          )     ,				dtype=numpy.floataa     ,				)

      # True output values for the given input values.
      _lowerCAmelCase 			=  numpy.array(([0], [1], [1], [0], [1], [0], [0], [1])     ,				dtype=numpy.floataa				)

      # Calling neural network class.
      _lowerCAmelCase 			=  TwoHiddenLayerNeuralNetwork(
          input_array=_SCREAMING_SNAKE_CASE     ,				output_array=_SCREAMING_SNAKE_CASE				)

      # Calling training function.
      # Set give_loss to True if you want to see loss in every iteration.
      neural_network.train(output=_SCREAMING_SNAKE_CASE     ,				iterations=1_0     ,				give_loss=_SCREAMING_SNAKE_CASE				)

      return neural_network.predict(numpy.array(([1, 1, 1])     ,				dtype=numpy.floataa				)				)


if __name__ == "__main__":
    example()