


"""simple docstring"""


import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class   _a      (      SCREAMING_SNAKE_CASE_		,		unittest.TestCase ):
    a_    :				Tuple             =    CodeGenTokenizer
    a_    :				List[str]             =    CodeGenTokenizerFast
    a_    :				Optional[int]             =    True
    a_    :				Union[str, Any]             =    {'add_prefix_space': True}
    a_    :				List[str]             =    False
    def      _UpperCamelCase       ( self							:					int				):
          super().setUp()

          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          lowerCamelCase__           =		[
              'l',
              'o',
              'w',
              'e',
              'r',
              's',
              't',
              'i',
              'd',
              'n',
              '\u0120',
              '\u0120l',
              '\u0120n',
              '\u0120lo',
              '\u0120low',
              'er',
              '\u0120lowest',
              '\u0120newer',
              '\u0120wider',
              '<unk>',
              '<|endoftext|>',
          ]
          lowerCamelCase__           =		dict(zip(SCREAMING_SNAKE_CASE__							,					range(len(SCREAMING_SNAKE_CASE__				)				)				)				)
          lowerCamelCase__           =		['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
          lowerCamelCase__           =		{'unk_token': '<unk>'}

          lowerCamelCase__           =		os.path.join(self.tmpdirname							,					VOCAB_FILES_NAMES['vocab_file']				)
          lowerCamelCase__           =		os.path.join(self.tmpdirname							,					VOCAB_FILES_NAMES['merges_file']				)
          with open(self.vocab_file							,					'w'							,					encoding='utf-8'				) as fp:
                fp.write(json.dumps(SCREAMING_SNAKE_CASE__				) + '\n'				)
          with open(self.merges_file							,					'w'							,					encoding='utf-8'				) as fp:
                fp.write('\n'.join(SCREAMING_SNAKE_CASE__				)				)
    def      _UpperCamelCase       ( self							:					Union[str, Any]							,					**SCREAMING_SNAKE_CASE__							:					Union[str, Any]				):
          kwargs.update(self.special_tokens_map				)
          return CodeGenTokenizer.from_pretrained(self.tmpdirname							,					**SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					List[str]							,					**SCREAMING_SNAKE_CASE__							:					str				):
          kwargs.update(self.special_tokens_map				)
          return CodeGenTokenizerFast.from_pretrained(self.tmpdirname							,					**SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					int							,					SCREAMING_SNAKE_CASE__							:					List[str]				):
          lowerCamelCase__           =		'lower newer'
          lowerCamelCase__           =		'lower newer'
          return input_text, output_text
    def      _UpperCamelCase       ( self							:					Union[str, Any]				):
          lowerCamelCase__           =		CodeGenTokenizer(self.vocab_file							,					self.merges_file							,					**self.special_tokens_map				)
          lowerCamelCase__           =		'lower newer'
          lowerCamelCase__           =		['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
          lowerCamelCase__           =		tokenizer.tokenize(SCREAMING_SNAKE_CASE__							,					add_prefix_space=SCREAMING_SNAKE_CASE__				)
          self.assertListEqual(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)

          lowerCamelCase__           =		tokens + [tokenizer.unk_token]
          lowerCamelCase__           =		[14, 15, 10, 9, 3, 2, 15, 19]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__				)							,					SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					str				):
          if not self.test_rust_tokenizer:
                return

          lowerCamelCase__           =		self.get_tokenizer()
          lowerCamelCase__           =		self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE__				)

          lowerCamelCase__           =		'lower newer'

          # Testing tokenization
          lowerCamelCase__           =		tokenizer.tokenize(SCREAMING_SNAKE_CASE__							,					add_prefix_space=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__				)
          self.assertListEqual(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)

          # Testing conversion to ids without special tokens
          lowerCamelCase__           =		tokenizer.encode(SCREAMING_SNAKE_CASE__							,					add_special_tokens=SCREAMING_SNAKE_CASE__							,					add_prefix_space=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		rust_tokenizer.encode(SCREAMING_SNAKE_CASE__							,					add_special_tokens=SCREAMING_SNAKE_CASE__				)
          self.assertListEqual(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)

          # Testing conversion to ids with special tokens
          lowerCamelCase__           =		self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		tokenizer.encode(SCREAMING_SNAKE_CASE__							,					add_prefix_space=SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		rust_tokenizer.encode(SCREAMING_SNAKE_CASE__				)
          self.assertListEqual(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)

          # Testing the unknown token
          lowerCamelCase__           =		tokens + [rust_tokenizer.unk_token]
          lowerCamelCase__           =		[14, 15, 10, 9, 3, 2, 15, 19]
          self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__				)							,					SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					List[Any]							,					*SCREAMING_SNAKE_CASE__							:					Dict							,					**SCREAMING_SNAKE_CASE__							:					str				):
          # It's very difficult to mix/test pretokenization with byte-level
          # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
          pass
    def      _UpperCamelCase       ( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]=15				):
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
                      lowerCamelCase__           =		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__				)

                      # Simple input
                      lowerCamelCase__           =		'This is a simple input'
                      lowerCamelCase__           =		['This is a simple input 1', 'This is a simple input 2']
                      lowerCamelCase__           =		('This is a simple input', 'This is a pair')
                      lowerCamelCase__           =		[
                          ('This is a simple input 1', 'This is a simple input 2'),
                          ('This is a simple pair 1', 'This is a simple pair 2'),
                      ]

                      # Simple input tests
                      self.assertRaises(SCREAMING_SNAKE_CASE__							,					tokenizer_r.encode							,					SCREAMING_SNAKE_CASE__							,					max_length=SCREAMING_SNAKE_CASE__							,					padding='max_length'				)

                      # Simple input
                      self.assertRaises(SCREAMING_SNAKE_CASE__							,					tokenizer_r.encode_plus							,					SCREAMING_SNAKE_CASE__							,					max_length=SCREAMING_SNAKE_CASE__							,					padding='max_length'				)

                      # Simple input
                      self.assertRaises(
                          SCREAMING_SNAKE_CASE__							,					tokenizer_r.batch_encode_plus							,					SCREAMING_SNAKE_CASE__							,					max_length=SCREAMING_SNAKE_CASE__							,					padding='max_length'							,					)

                      # Pair input
                      self.assertRaises(SCREAMING_SNAKE_CASE__							,					tokenizer_r.encode							,					SCREAMING_SNAKE_CASE__							,					max_length=SCREAMING_SNAKE_CASE__							,					padding='max_length'				)

                      # Pair input
                      self.assertRaises(SCREAMING_SNAKE_CASE__							,					tokenizer_r.encode_plus							,					SCREAMING_SNAKE_CASE__							,					max_length=SCREAMING_SNAKE_CASE__							,					padding='max_length'				)

                      # Pair input
                      self.assertRaises(
                          SCREAMING_SNAKE_CASE__							,					tokenizer_r.batch_encode_plus							,					SCREAMING_SNAKE_CASE__							,					max_length=SCREAMING_SNAKE_CASE__							,					padding='max_length'							,					)
    def      _UpperCamelCase       ( self							:					List[Any]				):
          lowerCamelCase__           =		CodeGenTokenizer.from_pretrained(self.tmpdirname							,					pad_token='<pad>'				)

          # Simple input
          lowerCamelCase__           =		'This is a simple input'
          lowerCamelCase__           =		['This is a simple input looooooooong', 'This is a simple input']
          lowerCamelCase__           =		('This is a simple input', 'This is a pair')
          lowerCamelCase__           =		[
              ('This is a simple input loooooong', 'This is a simple input'),
              ('This is a simple pair loooooong', 'This is a simple pair'),
          ]

          lowerCamelCase__           =		tokenizer.pad_token_id

          lowerCamelCase__           =		tokenizer(SCREAMING_SNAKE_CASE__							,					padding='max_length'							,					max_length=30							,					return_tensors='np'				)
          lowerCamelCase__           =		tokenizer(SCREAMING_SNAKE_CASE__							,					padding=SCREAMING_SNAKE_CASE__							,					truncate=SCREAMING_SNAKE_CASE__							,					return_tensors='np'				)
          lowerCamelCase__           =		tokenizer(*SCREAMING_SNAKE_CASE__							,					padding='max_length'							,					max_length=60							,					return_tensors='np'				)
          lowerCamelCase__           =		tokenizer(SCREAMING_SNAKE_CASE__							,					padding=SCREAMING_SNAKE_CASE__							,					truncate=SCREAMING_SNAKE_CASE__							,					return_tensors='np'				)

          # s
          # test single string max_length padding
          self.assertEqual(out_s['input_ids'].shape[-1]							,					30				)
          self.assertTrue(pad_token_id in out_s['input_ids']				)
          self.assertTrue(0 in out_s['attention_mask']				)

          # s2
          # test automatic padding
          self.assertEqual(out_sa['input_ids'].shape[-1]							,					33				)
          # long slice doesn't have padding
          self.assertFalse(pad_token_id in out_sa['input_ids'][0]				)
          self.assertFalse(0 in out_sa['attention_mask'][0]				)
          # short slice does have padding
          self.assertTrue(pad_token_id in out_sa['input_ids'][1]				)
          self.assertTrue(0 in out_sa['attention_mask'][1]				)

          # p
          # test single pair max_length padding
          self.assertEqual(out_p['input_ids'].shape[-1]							,					60				)
          self.assertTrue(pad_token_id in out_p['input_ids']				)
          self.assertTrue(0 in out_p['attention_mask']				)

          # p2
          # test automatic padding pair
          self.assertEqual(out_pa['input_ids'].shape[-1]							,					52				)
          # long slice pair doesn't have padding
          self.assertFalse(pad_token_id in out_pa['input_ids'][0]				)
          self.assertFalse(0 in out_pa['attention_mask'][0]				)
          # short slice pair does have padding
          self.assertTrue(pad_token_id in out_pa['input_ids'][1]				)
          self.assertTrue(0 in out_pa['attention_mask'][1]				)
    def      _UpperCamelCase       ( self							:					int				):
          lowerCamelCase__           =		'$$$'
          lowerCamelCase__           =		CodeGenTokenizer.from_pretrained(self.tmpdirname							,					bos_token=SCREAMING_SNAKE_CASE__							,					add_bos_token=SCREAMING_SNAKE_CASE__				)

          lowerCamelCase__           =		'This is a simple input'
          lowerCamelCase__           =		['This is a simple input 1', 'This is a simple input 2']

          lowerCamelCase__           =		tokenizer.bos_token_id

          lowerCamelCase__           =		tokenizer(SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		tokenizer(SCREAMING_SNAKE_CASE__				)

          self.assertEqual(out_s.input_ids[0]							,					SCREAMING_SNAKE_CASE__				)
          self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids				)				)

          lowerCamelCase__           =		tokenizer.decode(out_s.input_ids				)
          lowerCamelCase__           =		tokenizer.batch_decode(out_sa.input_ids				)

          self.assertEqual(decode_s.split()[0]							,					SCREAMING_SNAKE_CASE__				)
          self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa				)				)
    @slow
    def      _UpperCamelCase       ( self							:					Union[str, Any]				):
          lowerCamelCase__           =		CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'				)

          lowerCamelCase__           =		'\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
          lowerCamelCase__           =		'\nif len_a > len_b:      result = a\nelse:      result = b'

          lowerCamelCase__           =		tokenizer.encode(SCREAMING_SNAKE_CASE__				)
          lowerCamelCase__           =		['^#', re.escape('<|endoftext|>'				), '^\'\'\'', '^"""', '\n\n\n']
          lowerCamelCase__           =		tokenizer.decode(SCREAMING_SNAKE_CASE__							,					truncate_before_pattern=SCREAMING_SNAKE_CASE__				)
          self.assertEqual(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)


    def      _UpperCamelCase       ( self							:					Optional[int]				):
          pass







"""simple docstring"""


from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from typing import Generic, TypeVar

_snake_case          =							TypeVar("KEY")
_snake_case          =							TypeVar("VAL")



@dataclass(frozen=SCREAMING_SNAKE_CASE_		,		slots=SCREAMING_SNAKE_CASE_ )
class   _a      (      Generic[KEY, VAL] ):
    a_    :				KEY
    a_    :				VAL





class   _a      (      _Item ):
    def __init__( self							:					List[str]				):
          super().__init__(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)


    def __bool__( self							:					str				):
          return False





_snake_case          =							_DeletedItem()



class   _a      (      MutableMapping[KEY, VAL] ):
    def __init__( self							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					int = 8							,					SCREAMING_SNAKE_CASE__							:					float = 0.75				):
          lowerCamelCase__           =		initial_block_size
          lowerCamelCase__           =		[None] * initial_block_size
          assert 0.0 < capacity_factor < 1.0
          lowerCamelCase__           =		capacity_factor
          lowerCamelCase__           =		0
    def      _UpperCamelCase       ( self							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					KEY				):
          return hash(SCREAMING_SNAKE_CASE__				) % len(self._buckets				)
    def      _UpperCamelCase       ( self							:					str							,					SCREAMING_SNAKE_CASE__							:					int				):
          return (ind + 1) % len(self._buckets				)
    def      _UpperCamelCase       ( self							:					Union[str, Any]							,					SCREAMING_SNAKE_CASE__							:					int							,					SCREAMING_SNAKE_CASE__							:					KEY							,					SCREAMING_SNAKE_CASE__							:					VAL				):
          lowerCamelCase__           =		self._buckets[ind]
          if not stored:
                lowerCamelCase__           =		_Item(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)
                self._len += 1
                return True
          elif stored.key == key:
                lowerCamelCase__           =		_Item(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)
                return True
          else:
                return False
    def      _UpperCamelCase       ( self							:					Dict				):
          lowerCamelCase__           =		len(self._buckets				) * self._capacity_factor
          return len(self				) >= int(SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					int				):
          if len(self._buckets				) <= self._initial_block_size:
                return False
          lowerCamelCase__           =		len(self._buckets				) * self._capacity_factor / 2
          return len(self				) < limit
    def      _UpperCamelCase       ( self							:					List[str]							,					SCREAMING_SNAKE_CASE__							:					int				):
          lowerCamelCase__           =		self._buckets
          lowerCamelCase__           =		[None] * new_size
          lowerCamelCase__           =		0
          for item in old_buckets:
                if item:
                      self._add_item(item.key							,					item.val				)
    def      _UpperCamelCase       ( self							:					List[str]				):
          self._resize(len(self._buckets				) * 2				)
    def      _UpperCamelCase       ( self							:					Optional[int]				):
          self._resize(len(self._buckets				) // 2				)
    def      _UpperCamelCase       ( self							:					Any							,					SCREAMING_SNAKE_CASE__							:					KEY				):
          lowerCamelCase__           =		self._get_bucket_index(SCREAMING_SNAKE_CASE__				)
          for _ in range(len(self._buckets				)				):
                yield ind
                lowerCamelCase__           =		self._get_next_ind(SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					KEY							,					SCREAMING_SNAKE_CASE__							:					VAL				):
          for ind in self._iterate_buckets(SCREAMING_SNAKE_CASE__				):
                if self._try_set(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				):
                      break
    def __setitem__( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					KEY							,					SCREAMING_SNAKE_CASE__							:					VAL				):
          if self._is_full():
                self._size_up()

          self._add_item(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)
    def __delitem__( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					KEY				):
          for ind in self._iterate_buckets(SCREAMING_SNAKE_CASE__				):
                lowerCamelCase__           =		self._buckets[ind]
                if item is None:
                      raise KeyError(SCREAMING_SNAKE_CASE__				)
                if item is _deleted:
                      continue
                if item.key == key:
                      lowerCamelCase__           =		_deleted
                      self._len -= 1
                      break
          if self._is_sparse():
                self._size_down()
    def __getitem__( self							:					str							,					SCREAMING_SNAKE_CASE__							:					KEY				):
          for ind in self._iterate_buckets(SCREAMING_SNAKE_CASE__				):
                lowerCamelCase__           =		self._buckets[ind]
                if item is None:
                      break
                if item is _deleted:
                      continue
                if item.key == key:
                      return item.val
          raise KeyError(SCREAMING_SNAKE_CASE__				)
    def __len__( self							:					List[Any]				):
          return self._len
    def __iter__( self							:					Optional[int]				):
          yield from (item.key for item in self._buckets if item)


    def __repr__( self							:					str				):
          lowerCamelCase__           =		' ,'.join(
              F'{item.key}: {item.val}' for item in self._buckets if item				)
          return F'HashMap({val_string})'



