

import os
import shutil
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
from datasets import Dataset

from transformers.models.realm.configuration_realm import RealmConfig
from transformers.models.realm.retrieval_realm import _REALM_BLOCK_RECORDS_FILENAME, RealmRetriever
from transformers.models.realm.tokenization_realm import VOCAB_FILES_NAMES, RealmTokenizer



class       __snake_case  (    lowerCAmelCase__       ):

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						tempfile.mkdtemp()
      SCREAMING_SNAKE_CASE_									=						5

      # Realm tok
      SCREAMING_SNAKE_CASE_									=						[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'test',
          'question',
          'this',
          'is',
          'the',
          'first',
          'second',
          'third',
          'fourth',
          'fifth',
          'record',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      SCREAMING_SNAKE_CASE_									=						os.path.join(self.tmpdirname						,		'realm_tokenizer')
      os.makedirs(_A						,		exist_ok=_A)
      SCREAMING_SNAKE_CASE_									=						os.path.join(_A						,		VOCAB_FILES_NAMES['vocab_file'])
      with open(self.vocab_file						,		'w'						,		encoding='utf-8') as vocab_writer:
          vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))

      SCREAMING_SNAKE_CASE_									=						os.path.join(self.tmpdirname						,		'realm_block_records')
      os.makedirs(_A						,		exist_ok=_A)

  def 						lowerCAmelCase__     (					self):
      return RealmTokenizer.from_pretrained(os.path.join(self.tmpdirname						,		'realm_tokenizer'))

  def 						lowerCAmelCase__     (					self):
      shutil.rmtree(self.tmpdirname)

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						RealmConfig(num_block_records=self.num_block_records)
      return config

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						Dataset.from_dict(
          {
              'id': ['0', '1'],
              'question': ['foo', 'bar'],
              'answers': [['Foo', 'Bar'], ['Bar']],
          })
      return dataset

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						np.array(
          [
              b'This is the first record',
              b'This is the second record',
              b'This is the third record',
              b'This is the fourth record',
              b'This is the fifth record',
              b'This is a longer longer longer record',
          ]						,		dtype=_A						,		)
      return block_records

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						RealmRetriever(
          block_records=self.get_dummy_block_records()						,		tokenizer=self.get_tokenizer()						,		)
      return retriever

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.get_config()
      SCREAMING_SNAKE_CASE_									=						self.get_dummy_retriever()
      SCREAMING_SNAKE_CASE_									=						retriever.tokenizer

      SCREAMING_SNAKE_CASE_									=						np.array([0, 3]						,		dtype='long')
      SCREAMING_SNAKE_CASE_									=						tokenizer(['Test question']).input_ids
      SCREAMING_SNAKE_CASE_									=						tokenizer(
          ['the fourth']						,		add_special_tokens=_A						,		return_token_type_ids=_A						,		return_attention_mask=_A						,		).input_ids
      SCREAMING_SNAKE_CASE_									=						config.reader_seq_len

      SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_									=						retriever(
          _A						,		_A						,		answer_ids=_A						,		max_length=_A						,		return_tensors='np')

      self.assertEqual(len(_A)						,		2)
      self.assertEqual(len(_A)						,		2)
      self.assertEqual(len(_A)						,		2)
      self.assertEqual(concat_inputs.input_ids.shape						,		(2, 10))
      self.assertEqual(concat_inputs.attention_mask.shape						,		(2, 10))
      self.assertEqual(concat_inputs.token_type_ids.shape						,		(2, 10))
      self.assertEqual(concat_inputs.special_tokens_mask.shape						,		(2, 10))
      self.assertEqual(
          tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[0])						,		['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'first', 'record', '[SEP]']						,		)
      self.assertEqual(
          tokenizer.convert_ids_to_tokens(concat_inputs.input_ids[1])						,		['[CLS]', 'test', 'question', '[SEP]', 'this', 'is', 'the', 'fourth', 'record', '[SEP]']						,		)

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.get_config()
      SCREAMING_SNAKE_CASE_									=						self.get_dummy_retriever()
      SCREAMING_SNAKE_CASE_									=						retriever.tokenizer

      SCREAMING_SNAKE_CASE_									=						np.array([0, 3, 5]						,		dtype='long')
      SCREAMING_SNAKE_CASE_									=						tokenizer(['Test question']).input_ids
      SCREAMING_SNAKE_CASE_									=						tokenizer(
          ['the fourth', 'longer longer']						,		add_special_tokens=_A						,		return_token_type_ids=_A						,		return_attention_mask=_A						,		).input_ids
      SCREAMING_SNAKE_CASE_									=						config.reader_seq_len

      SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_									=						retriever(
          _A						,		_A						,		answer_ids=_A						,		max_length=_A						,		return_tensors='np')

      self.assertEqual([False, True, True]						,		_A)
      self.assertEqual([[-1, -1, -1], [6, -1, -1], [6, 7, 8]]						,		_A)
      self.assertEqual([[-1, -1, -1], [7, -1, -1], [7, 8, 9]]						,		_A)


  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.get_dummy_retriever()
      retriever.save_pretrained(os.path.join(self.tmpdirname						,		'realm_block_records'))

      # Test local path
      SCREAMING_SNAKE_CASE_									=						retriever.from_pretrained(os.path.join(self.tmpdirname						,		'realm_block_records'))
      self.assertEqual(retriever.block_records[0]						,		b'This is the first record')

      # Test mocked remote path
      with patch('transformers.models.realm.retrieval_realm.hf_hub_download') as mock_hf_hub_download:
          SCREAMING_SNAKE_CASE_									=						os.path.join(
              os.path.join(self.tmpdirname						,		'realm_block_records')						,		_REALM_BLOCK_RECORDS_FILENAME)
          SCREAMING_SNAKE_CASE_									=						RealmRetriever.from_pretrained('google/realm-cc-news-pretrained-openqa')

      self.assertEqual(retriever.block_records[0]						,		b'This is the first record')





from multiprocessing import Lock, Pipe, Process

# lock used to ensure that two processes do not access a pipe at the same time
UpperCamelCase__  :      int			=    Lock()


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		str ,   _SCREAMING_SNAKE_CASE					:		Union[str, Any] ,   _SCREAMING_SNAKE_CASE					:		Dict ,   _SCREAMING_SNAKE_CASE					:		List[Any] ,   _SCREAMING_SNAKE_CASE					:		Optional[int] ,   _SCREAMING_SNAKE_CASE					:		Optional[int] ,   _SCREAMING_SNAKE_CASE					:		Any       ):





    """simple docstring"""


    global process_lock

    # we perform n swaps since after n swaps we know we are sorted
    # we *could* stop early if we are sorted already, but it takes as long to
    # find out we are sorted as it does to sort the list with this algorithm
    for i in range(0 ,   10       ):
        if (i + position) % 2 == 0 and r_send is not None:
            # send your value to your right neighbor
            process_lock.acquire()
            r_send[1].send(_SCREAMING_SNAKE_CASE       )
            process_lock.release()

            # receive your right neighbor's value
            process_lock.acquire()
            SCREAMING_SNAKE_CASE_									=						rr_cv[0].recv()
            process_lock.release()

            # take the lower value since you are on the left
            SCREAMING_SNAKE_CASE_									=						min(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
        elif (i + position) % 2 != 0 and l_send is not None:
            # send your value to your left neighbor
            process_lock.acquire()
            l_send[1].send(_SCREAMING_SNAKE_CASE       )
            process_lock.release()

            # receive your left neighbor's value
            process_lock.acquire()
            SCREAMING_SNAKE_CASE_									=						lr_cv[0].recv()
            process_lock.release()

            # take the higher value since you are on the right
            SCREAMING_SNAKE_CASE_									=						max(_SCREAMING_SNAKE_CASE ,   _SCREAMING_SNAKE_CASE       )
    # after all swaps are performed, send the values back to main
    result_pipe[1].send(_SCREAMING_SNAKE_CASE       )


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Union[str, Any]       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						[]
    SCREAMING_SNAKE_CASE_									=						[]
    # initialize the list of pipes where the values will be retrieved
    for _ in arr:
        result_pipe.append(Pipe()       )
    # creates the processes
    # the first and last process only have one neighbor so they are made outside
    # of the loop
    SCREAMING_SNAKE_CASE_									=						Pipe()
    SCREAMING_SNAKE_CASE_									=						Pipe()
    process_array_.append(
        Process(
            target=_SCREAMING_SNAKE_CASE ,   args=(0, arr[0], None, temp_rs, None, temp_rr, result_pipe[0]) ,   )       )
    SCREAMING_SNAKE_CASE_									=						temp_rs
    SCREAMING_SNAKE_CASE_									=						temp_rr

    for i in range(1 ,   len(_SCREAMING_SNAKE_CASE       ) - 1       ):
        SCREAMING_SNAKE_CASE_									=						Pipe()
        SCREAMING_SNAKE_CASE_									=						Pipe()
        process_array_.append(
            Process(
                target=_SCREAMING_SNAKE_CASE ,   args=(i, arr[i], temp_ls, temp_rs, temp_lr, temp_rr, result_pipe[i]) ,   )       )
        SCREAMING_SNAKE_CASE_									=						temp_rs
        SCREAMING_SNAKE_CASE_									=						temp_rr

    process_array_.append(
        Process(
            target=_SCREAMING_SNAKE_CASE ,   args=(
                len(_SCREAMING_SNAKE_CASE       ) - 1,
                arr[len(_SCREAMING_SNAKE_CASE       ) - 1],
                temp_ls,
                None,
                temp_lr,
                None,
                result_pipe[len(_SCREAMING_SNAKE_CASE       ) - 1],
            ) ,   )       )

    # start the processes
    for p in process_array_:
        p.start()

    # wait for the processes to end and write their values to the list
    for p in range(0 ,   len(_SCREAMING_SNAKE_CASE       )       ):
        SCREAMING_SNAKE_CASE_									=						result_pipe[p][0].recv()
        process_array_[p].join()
    return arr


def 					_UpperCAmelCase       (  ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						list(range(10 ,   0 ,   -1       )       )
    print('Initial List'       )
    print(*_SCREAMING_SNAKE_CASE       )
    SCREAMING_SNAKE_CASE_									=						odd_even_transposition(_SCREAMING_SNAKE_CASE       )
    print('Sorted List\n'       )
    print(*_SCREAMING_SNAKE_CASE       )


if __name__ == "__main__":
       main()


