


from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCamelCase				:  str      					=							logging.get_logger(__name__)


__UpperCamelCase				:  Tuple      					=							{
    'nielsr/canine-s': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
__UpperCamelCase				:  Optional[int]      					=							111_4112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
__UpperCamelCase				:  List[str]      					=							0

__UpperCamelCase				:  str      					=							0xe000
__UpperCamelCase				:  Union[str, Any]      					=							0xe001
__UpperCamelCase				:  Any      					=							0xe002
__UpperCamelCase				:  Dict      					=							0xe003
__UpperCamelCase				:  Tuple      					=							0xe004

# Maps special codepoints to human-readable names.
__UpperCamelCase				:  Dict[int, str]      					=							{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
__UpperCamelCase				:  Dict[str, int]      					=							{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     Any			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





    def __init__(	self				:   List[str]       ,   __snake_case				:   Optional[Any]=chr(__snake_case	)       ,   __snake_case				:   Optional[int]=chr(__snake_case	)       ,   __snake_case				:   Dict=chr(__snake_case	)       ,   __snake_case				:   Tuple=chr(__snake_case	)       ,   __snake_case				:   Dict=chr(__snake_case	)       ,   __snake_case				:   Tuple=chr(__snake_case	)       ,   __snake_case				:   List[Any]=False       ,   __snake_case				:   Optional[int]=2_048       ,   **__snake_case				:   str       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Dict						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else bos_token
       UpperCAmelCase_							:      int						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else eos_token
       UpperCAmelCase_							:      Optional[Any]						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else sep_token
       UpperCAmelCase_							:      List[Any]						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else cls_token
       UpperCAmelCase_							:      str						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       UpperCAmelCase_							:      Dict						=   AddedToken(__snake_case       ,   lstrip=__snake_case       ,   rstrip=__snake_case	) if isinstance(__snake_case       ,   __snake_case	) else mask_token

       super().__init__(
           bos_token=__snake_case       ,   eos_token=__snake_case       ,   sep_token=__snake_case       ,   cls_token=__snake_case       ,   pad_token=__snake_case       ,   mask_token=__snake_case       ,   add_prefix_space=__snake_case       ,   model_max_length=__snake_case       ,   **__snake_case       ,   )

       # Creates a mapping for looking up the IDs of special symbols.
       UpperCAmelCase_							:      Dict[str, int]						=   {}
       for codepoint, name in SPECIAL_CODEPOINTS.items():
          UpperCAmelCase_							:      List[str]						=   codepoint

       # Creates a mapping for looking up the string forms of special symbol IDs.
       UpperCAmelCase_							:      Dict[int, str]						=   {
           codepoint: name for name, codepoint in self._special_codepoints.items()
       }

       UpperCAmelCase_							:      Dict						=   UNICODE_VOCAB_SIZE
       UpperCAmelCase_							:      Optional[int]						=   len(self._special_codepoints	)





    @property
    def 					_lowerCamelCase (	self				:   int	):





       '''simple docstring'''
       return self._unicode_vocab_size





    def 					_lowerCamelCase (	self				:   Optional[int]       ,   __snake_case				:   str	):





       '''simple docstring'''
       return list(__snake_case	)





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   str	):





       '''simple docstring'''
       try:
          return ord(__snake_case	)
       except TypeError:
          raise ValueError(f'''invalid token: \'{token}\''''	)





    def 					_lowerCamelCase (	self				:   str       ,   __snake_case				:   int	):





       '''simple docstring'''
       try:
          if index in SPECIAL_CODEPOINTS:
             return SPECIAL_CODEPOINTS[index]
          return chr(__snake_case	)
       except TypeError:
          raise ValueError(f'''invalid id: {index}'''	)





    def 					_lowerCamelCase (	self				:   Tuple       ,   __snake_case				:   Dict	):





       '''simple docstring'''
       return "".join(__snake_case	)





    def 					_lowerCamelCase (	self				:   List[Any]       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   [self.sep_token_id]
       UpperCAmelCase_							:      Any						=   [self.cls_token_id]

       UpperCAmelCase_							:      Union[str, Any]						=   cls + token_ids_a + sep
       if token_ids_a is not None:
          result += token_ids_a + sep
       return result





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None       ,   __snake_case				:   bool = False	):





       '''simple docstring'''
       if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__snake_case       ,   token_ids_a=__snake_case       ,   already_has_special_tokens=__snake_case	)

       UpperCAmelCase_							:      int						=   [1] + ([0] * len(__snake_case	)) + [1]
       if token_ids_a is not None:
          result += ([0] * len(__snake_case	)) + [1]
       return result





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[Any]						=   [self.sep_token_id]
       UpperCAmelCase_							:      Dict						=   [self.cls_token_id]

       UpperCAmelCase_							:      List[Any]						=   len(cls + token_ids_a + sep	) * [0]
       if token_ids_a is not None:
          result += len(token_ids_a + sep	) * [1]
       return result





    def 					_lowerCamelCase (	self				:   Union[str, Any]       ,   __snake_case				:   str       ,   __snake_case				:   Optional[str] = None	):





       '''simple docstring'''
       return ()



# This code is adapted from OpenAI's release
# https://github.com/openai/human-eval/blob/master/human_eval/execution.py

import contextlib
import faulthandler
import io
import multiprocessing
import os
import platform
import signal
import tempfile
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase			,       __lowercase       ):
   UpperCAmelCase_							:      int						=   multiprocessing.Manager()
   UpperCAmelCase_							:      Union[str, Any]						=   manager.list()

   UpperCAmelCase_							:      int						=   multiprocessing.Process(target=__lowercase			,       args=(check_program, result, timeout)       )
   p.start()
   p.join(timeout=timeout + 1       )
   if p.is_alive():
      p.kill()

   if not result:
      result.append('''timed out'''       )

   return {
       "task_id": task_id,
       "passed": result[0] == "passed",
       "result": result[0],
       "completion_id": completion_id,
   }
def      snake_case_						( __lowercase			,       __lowercase			,       __lowercase       ):
   with create_tempdir():
      # These system calls are needed when cleaning up tempdir.
      import os
      import shutil

      UpperCAmelCase_							:      str						=   shutil.rmtree
      UpperCAmelCase_							:      Tuple						=   os.rmdir
      UpperCAmelCase_							:      Dict						=   os.chdir

      # Disable functionalities that can make destructive changes to the test.
      reliability_guard()

      # Run program.
      try:
         UpperCAmelCase_							:      Optional[int]						=   {}
         with swallow_io():
            with time_limit(__lowercase       ):
               exec(__lowercase			,       __lowercase       )
         result.append('''passed'''       )
      except TimeoutException:
         result.append('''timed out'''       )
      except BaseException as e:
         result.append(F'''failed: {e}'''       )

      # Needed for cleaning up.
      UpperCAmelCase_							:      Optional[int]						=   rmtree
      UpperCAmelCase_							:      Optional[Any]						=   rmdir
      UpperCAmelCase_							:      Optional[Any]						=   chdir
@contextlib.contextmanager
def      snake_case_						( __lowercase       ):
   def signal_handler(__lowercase			,       __lowercase       ):
      raise TimeoutException('''Timed out!'''       )

   signal.setitimer(signal.ITIMER_REAL			,       __lowercase       )
   signal.signal(signal.SIGALRM			,       __lowercase       )
   try:
      yield
   finally:
      signal.setitimer(signal.ITIMER_REAL			,       0       )
@contextlib.contextmanager
def      snake_case_						( ):
   UpperCAmelCase_							:      Optional[Any]						=   WriteOnlyStringIO()
   with contextlib.redirect_stdout(__lowercase       ):
      with contextlib.redirect_stderr(__lowercase       ):
         with redirect_stdin(__lowercase       ):
            yield
@contextlib.contextmanager
def      snake_case_						( ):
   with tempfile.TemporaryDirectory() as dirname:
      with chdir(__lowercase       ):
         yield dirname





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    pass





class  lowerCAmelCase__(    io.StringIO						):






    '''simple docstring'''





    def 					_lowerCamelCase (	self				:   Dict       ,   *__snake_case				:   List[Any]       ,   **__snake_case				:   int	):





       '''simple docstring'''
       raise OSError





    def 					_lowerCamelCase (	self				:   Dict       ,   *__snake_case				:   int       ,   **__snake_case				:   Any	):





       '''simple docstring'''
       raise OSError





    def 					_lowerCamelCase (	self				:   int       ,   *__snake_case				:   List[str]       ,   **__snake_case				:   Optional[Any]	):





       '''simple docstring'''
       raise OSError





    def 					_lowerCamelCase (	self				:   Union[str, Any]       ,   *__snake_case				:   Optional[Any]       ,   **__snake_case				:   List[Any]	):





       '''simple docstring'''
       return False





class  lowerCAmelCase__(    contextlib._RedirectStream						):  # type: ignore






    '''simple docstring'''





    A_  :     Optional[Any]			= 'stdin'





@contextlib.contextmanager
def      snake_case_						( __lowercase       ):
   if root == ".":
      yield
      return
   UpperCAmelCase_							:      Tuple						=   os.getcwd()
   os.chdir(__lowercase       )
   try:
      yield
   except BaseException as exc:
      raise exc
   finally:
      os.chdir(__lowercase       )


def      snake_case_						( __lowercase=None       ):

   if maximum_memory_bytes is not None:
      import resource

      resource.setrlimit(resource.RLIMIT_AS			,       (maximum_memory_bytes, maximum_memory_bytes)       )
      resource.setrlimit(resource.RLIMIT_DATA			,       (maximum_memory_bytes, maximum_memory_bytes)       )
      if not platform.uname().system == "Darwin":
         resource.setrlimit(resource.RLIMIT_STACK			,       (maximum_memory_bytes, maximum_memory_bytes)       )

   faulthandler.disable()

   import builtins

   UpperCAmelCase_							:      Any						=   None
   UpperCAmelCase_							:      Any						=   None

   import os

   UpperCAmelCase_							:      Union[str, Any]						=   '''1'''

   UpperCAmelCase_							:      List[str]						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      Any						=   None
   UpperCAmelCase_							:      Optional[int]						=   None
   UpperCAmelCase_							:      List[str]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      Union[str, Any]						=   None
   UpperCAmelCase_							:      str						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      int						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      int						=   None
   UpperCAmelCase_							:      Union[str, Any]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      Union[str, Any]						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      Any						=   None

   import shutil

   UpperCAmelCase_							:      Union[str, Any]						=   None
   UpperCAmelCase_							:      Dict						=   None
   UpperCAmelCase_							:      Tuple						=   None

   import subprocess

   UpperCAmelCase_							:      Dict						=   None  # type: ignore

   UpperCAmelCase_							:      Union[str, Any]						=   None

   import sys

   UpperCAmelCase_							:      Optional[Any]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      List[Any]						=   None
   UpperCAmelCase_							:      List[str]						=   None
   UpperCAmelCase_							:      Dict						=   None