


import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin





class  lowerCAmelCase__(    snake_case__    ,    unittest.TestCase						):






    '''simple docstring'''





    A_  :     int			= ProphetNetTokenizer
    A_  :     Union[str, Any]			= False





    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       super().setUp()

       UpperCAmelCase_							:      Tuple						=   [
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       UpperCAmelCase_							:      List[Any]						=   os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES['''vocab_file''']	)
       with open(self.vocab_file       ,   '''w'''       ,   encoding='''utf-8'''	) as vocab_writer:
          vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)





    def 					_lowerCamelCase (	self				:   List[str]       ,   __snake_case				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   '''UNwant\u00E9d,running'''
       UpperCAmelCase_							:      List[str]						=   '''unwanted, running'''
       return input_text, output_text





    def 					_lowerCamelCase (	self				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[str]						=   self.tokenizer_class(self.vocab_file	)

       UpperCAmelCase_							:      str						=   tokenizer.tokenize('''UNwant\u00E9d,running'''	)
       self.assertListEqual(__snake_case       ,   ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']	)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case	)       ,   [9, 6, 7, 12, 10, 11]	)





    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       UpperCAmelCase_							:      Tuple						=   BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''	)       ,   ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']	)





    def 					_lowerCamelCase (	self				:   List[Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   BasicTokenizer(do_lower_case=__snake_case	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)       ,   ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)       ,   ['''hello''']	)





    def 					_lowerCamelCase (	self				:   Optional[int]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Tuple						=   BasicTokenizer(do_lower_case=__snake_case       ,   strip_accents=__snake_case	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)       ,   ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)       ,   ['''h\u00E9llo''']	)





    def 					_lowerCamelCase (	self				:   List[Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Dict						=   BasicTokenizer(do_lower_case=__snake_case       ,   strip_accents=__snake_case	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)       ,   ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)       ,   ['''hello''']	)





    def 					_lowerCamelCase (	self				:   Optional[int]	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   BasicTokenizer(do_lower_case=__snake_case	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)       ,   ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']	)
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''	)       ,   ['''hello''']	)





    def 					_lowerCamelCase (	self				:   Union[str, Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   BasicTokenizer(do_lower_case=__snake_case	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''	)       ,   ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)





    def 					_lowerCamelCase (	self				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      Tuple						=   BasicTokenizer(do_lower_case=__snake_case       ,   strip_accents=__snake_case	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)       ,   ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)





    def 					_lowerCamelCase (	self				:   Optional[int]	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   BasicTokenizer(do_lower_case=__snake_case       ,   strip_accents=__snake_case	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''	)       ,   ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']	)





    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[str]						=   BasicTokenizer(do_lower_case=__snake_case       ,   never_split=['''[UNK]''']	)

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''	)       ,   ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']	)





    def 					_lowerCamelCase (	self				:   List[str]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Tuple						=   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       UpperCAmelCase_							:      Any						=   {}
       for i, token in enumerate(__snake_case	):
          UpperCAmelCase_							:      List[str]						=   i
       UpperCAmelCase_							:      List[Any]						=   WordpieceTokenizer(vocab=__snake_case       ,   unk_token='''[UNK]'''	)

       self.assertListEqual(tokenizer.tokenize(''''''	)       ,   []	)

       self.assertListEqual(tokenizer.tokenize('''unwanted running'''	)       ,   ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']	)

       self.assertListEqual(tokenizer.tokenize('''unwantedX running'''	)       ,   ['''[UNK]''', '''runn''', '''##ing''']	)





    @require_torch
    def 					_lowerCamelCase (	self				:   List[Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''	)

       UpperCAmelCase_							:      int						=   ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
       UpperCAmelCase_							:      Optional[int]						=   [1_037, 2_146, 20_423, 2_005, 7_680, 7_849, 3_989, 1_012, 102]
       UpperCAmelCase_							:      str						=   tokenizer(__snake_case       ,   padding=__snake_case       ,   return_tensors='''pt'''	)
       self.assertIsInstance(__snake_case       ,   __snake_case	)
       UpperCAmelCase_							:      int						=   list(batch.input_ids.numpy()[0]	)
       self.assertListEqual(__snake_case       ,   __snake_case	)

       self.assertEqual((2, 9)       ,   batch.input_ids.shape	)
       self.assertEqual((2, 9)       ,   batch.attention_mask.shape	)





    def 					_lowerCamelCase (	self				:   Any	):





       '''simple docstring'''
       self.assertTrue(_is_whitespace(''' '''	)	)
       self.assertTrue(_is_whitespace('''\t'''	)	)
       self.assertTrue(_is_whitespace('''\r'''	)	)
       self.assertTrue(_is_whitespace('''\n'''	)	)
       self.assertTrue(_is_whitespace('''\u00A0'''	)	)

       self.assertFalse(_is_whitespace('''A'''	)	)
       self.assertFalse(_is_whitespace('''-'''	)	)





    def 					_lowerCamelCase (	self				:   List[Any]	):





       '''simple docstring'''
       self.assertTrue(_is_control('''\u0005'''	)	)

       self.assertFalse(_is_control('''A'''	)	)
       self.assertFalse(_is_control(''' '''	)	)
       self.assertFalse(_is_control('''\t'''	)	)
       self.assertFalse(_is_control('''\r'''	)	)





    def 					_lowerCamelCase (	self				:   List[Any]	):





       '''simple docstring'''
       self.assertTrue(_is_punctuation('''-'''	)	)
       self.assertTrue(_is_punctuation('''$'''	)	)
       self.assertTrue(_is_punctuation('''`'''	)	)
       self.assertTrue(_is_punctuation('''.'''	)	)

       self.assertFalse(_is_punctuation('''A'''	)	)
       self.assertFalse(_is_punctuation(''' '''	)	)





    @slow
    def 					_lowerCamelCase (	self				:   Optional[int]	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''	)

       UpperCAmelCase_							:      int						=   tokenizer.encode('''sequence builders'''       ,   add_special_tokens=__snake_case	)
       UpperCAmelCase_							:      List[str]						=   tokenizer.encode('''multi-sequence build'''       ,   add_special_tokens=__snake_case	)

       UpperCAmelCase_							:      int						=   tokenizer.build_inputs_with_special_tokens(__snake_case	)
       UpperCAmelCase_							:      List[Any]						=   tokenizer.build_inputs_with_special_tokens(__snake_case       ,   __snake_case	)

       assert encoded_sentence == text + [102]
       assert encoded_pair == text + [102] + text_a + [102]



# This code is adapted from OpenAI's release
# https://github.com/openai/human-eval/blob/master/human_eval/execution.py

import contextlib
import faulthandler
import io
import multiprocessing
import os
import platform
import signal
import tempfile
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase			,       __lowercase       ):
   UpperCAmelCase_							:      int						=   multiprocessing.Manager()
   UpperCAmelCase_							:      Union[str, Any]						=   manager.list()

   UpperCAmelCase_							:      int						=   multiprocessing.Process(target=__lowercase			,       args=(check_program, result, timeout)       )
   p.start()
   p.join(timeout=timeout + 1       )
   if p.is_alive():
      p.kill()

   if not result:
      result.append('''timed out'''       )

   return {
       "task_id": task_id,
       "passed": result[0] == "passed",
       "result": result[0],
       "completion_id": completion_id,
   }
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase       ):
   with create_tempdir():
      # These system calls are needed when cleaning up tempdir.
      import os
      import shutil

      UpperCAmelCase_							:      str						=   shutil.rmtree
      UpperCAmelCase_							:      Tuple						=   os.rmdir
      UpperCAmelCase_							:      Dict						=   os.chdir

      # Disable functionalities that can make destructive changes to the test.
      reliability_guard()

      # Run program.
      try:
         UpperCAmelCase_							:      Optional[int]						=   {}
         with swallow_io():
            with time_limit(__lowercase       ):
               exec(__lowercase			,       __lowercase       )
         result.append('''passed'''       )
      except TimeoutException:
         result.append('''timed out'''       )
      except BaseException as e:
         result.append(F'''failed: {e}'''       )

      # Needed for cleaning up.
      UpperCAmelCase_							:      Optional[int]						=   rmtree
      UpperCAmelCase_							:      Optional[Any]						=   rmdir
      UpperCAmelCase_							:      Optional[Any]						=   chdir
@contextlib.contextmanager
def      snake_case_						( __lowercase       ):
   def signal_handler(__lowercase			,       __lowercase       ):
      raise TimeoutException('''Timed out!'''       )

   signal.setitimer(signal.ITIMER_REAL			,       __lowercase       )
   signal.signal(signal.SIGALRM			,       __lowercase       )
   try:
      yield
   finally:
      signal.setitimer(signal.ITIMER_REAL			,       0       )
@contextlib.contextmanager
def      snake_case_						( ):
   UpperCAmelCase_							:      Optional[Any]						=   WriteOnlyStringIO()
   with contextlib.redirect_stdout(__lowercase       ):
      with contextlib.redirect_stderr(__lowercase       ):
         with redirect_stdin(__lowercase       ):
            yield
@contextlib.contextmanager
def      snake_case_						( ):
   with tempfile.TemporaryDirectory() as dirname:
      with chdir(__lowercase       ):
         yield dirname





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    pass





class  lowerCAmelCase__(    io.StringIO						):






    '''simple docstring'''





    def 					_lowerCamelCase (	self				:   Dict       ,   *__snake_case				:   List[Any]       ,   **__snake_case				:   int	):





       '''simple docstring'''
       raise OSError





    def 					_lowerCamelCase (	self				:   Dict       ,   *__snake_case				:   int       ,   **__snake_case				:   Any	):





       '''simple docstring'''
       raise OSError





    def 					_lowerCamelCase (	self				:   int       ,   *__snake_case				:   List[str]       ,   **__snake_case				:   Optional[Any]	):





       '''simple docstring'''
       raise OSError





    def 					_lowerCamelCase (	self				:   Union[str, Any]       ,   *__snake_case				:   Optional[Any]       ,   **__snake_case				:   List[Any]	):





       '''simple docstring'''
       return False





class  lowerCAmelCase__(    contextlib._RedirectStream						):  # type: ignore






    '''simple docstring'''





    A_  :     Optional[Any]			= 'stdin'





@contextlib.contextmanager
def      snake_case_						( __lowercase       ):
   if root == ".":
      yield
      return
   UpperCAmelCase_							:      Tuple						=   os.getcwd()
   os.chdir(__lowercase       )
   try:
      yield
   except BaseException as exc:
      raise exc
   finally:
      os.chdir(__lowercase       )


def      snake_case_						( __lowercase=None       ):

   if maximum_memory_bytes is not None:
      import resource

      resource.setrlimit(resource.RLIMIT_AS			,       (maximum_memory_bytes, maximum_memory_bytes)       )
      resource.setrlimit(resource.RLIMIT_DATA			,       (maximum_memory_bytes, maximum_memory_bytes)       )
      if not platform.uname().system == "Darwin":
         resource.setrlimit(resource.RLIMIT_STACK			,       (maximum_memory_bytes, maximum_memory_bytes)       )

   faulthandler.disable()

   import builtins

   UpperCAmelCase_							:      Any						=   None
   UpperCAmelCase_							:      Any						=   None

   import os

   UpperCAmelCase_							:      Union[str, Any]						=   '''1'''

   UpperCAmelCase_							:      List[str]						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      Any						=   None
   UpperCAmelCase_							:      Optional[int]						=   None
   UpperCAmelCase_							:      List[str]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      Union[str, Any]						=   None
   UpperCAmelCase_							:      str						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      int						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      int						=   None
   UpperCAmelCase_							:      Union[str, Any]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      Union[str, Any]						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      Any						=   None

   import shutil

   UpperCAmelCase_							:      Union[str, Any]						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      Tuple						=   None

   import subprocess

   UpperCAmelCase_							:      Dict						=   None  # type: ignore

   UpperCAmelCase_							:      Union[str, Any]						=   None

   import sys

   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      List[str]						=   None
   UpperCAmelCase_							:      Dict						=   None