"""simple docstring"""




import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a   :	Union[str, Any] 					=     logging.get_logger(__name__)

_a   :	Tuple 					=     {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_a   :	Optional[Any] 					=     {
    'vocab_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'},
    'merges_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'},
}

_a   :	List[str] 					=     {
    'ctrl': 256,
}

_a   :	List[Any] 					=     {
    'Pregnancy': 168_629,
    'Christianity': 7_675,
    'Explain': 106_423,
    'Fitness': 63_440,
    'Saving': 63_163,
    'Ask': 27_171,
    'Ass': 95_985,
    'Joke': 163_509,
    'Questions': 45_622,
    'Thoughts': 49_605,
    'Retail': 52_342,
    'Feminism': 164_338,
    'Writing': 11_992,
    'Atheism': 192_263,
    'Netflix': 48_616,
    'Computing': 39_639,
    'Opinion': 43_213,
    'Alone': 44_967,
    'Funny': 58_917,
    'Gaming': 40_358,
    'Human': 4_088,
    'India': 1_331,
    'Joker': 77_138,
    'Diet': 36_206,
    'Legal': 11_859,
    'Norman': 4_939,
    'Tip': 72_689,
    'Weight': 52_343,
    'Movies': 46_273,
    'Running': 23_425,
    'Science': 2_090,
    'Horror': 37_793,
    'Confession': 60_572,
    'Finance': 12_250,
    'Politics': 16_360,
    'Scary': 191_985,
    'Support': 12_654,
    'Technologies': 32_516,
    'Teenage': 66_160,
    'Event': 32_769,
    'Learned': 67_460,
    'Notion': 182_770,
    'Wikipedia': 37_583,
    'Books': 6_665,
    'Extract': 76_050,
    'Confessions': 102_701,
    'Conspiracy': 75_932,
    'Links': 63_674,
    'Narcissus': 150_425,
    'Relationship': 54_766,
    'Relationships': 134_796,
    'Reviews': 41_671,
    'News': 4_256,
    'Translation': 26_820,
    'multilingual': 128_406,
}





def 							SCREAMING_SNAKE_CASE  (    _lowerCamelCase		: List[Any]			) ->   Optional[int]:
  _lowerCAmelCase		: str						    =  set()
  _lowerCAmelCase		: Any						    =  word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)			)
    _lowerCAmelCase		: List[str]						    =  char

  _lowerCAmelCase		: str						    =  set(_lowerCamelCase			)
  return pairs






class 		__A							(				SCREAMING_SNAKE_CASE_	):
     _UpperCamelCase							:    List[str]	      =	VOCAB_FILES_NAMES
     _UpperCamelCase							:    Tuple	      =	PRETRAINED_VOCAB_FILES_MAP
     _UpperCamelCase							:    int	      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _UpperCamelCase							:    int	      =	CONTROL_CODES






     def __init__(	self       ,  a__       ,  a__       ,  a__="<unk>"       ,  **a__						):
       super().__init__(unk_token=a__       ,  **a__						)

       with open(a__       ,  encoding="""utf-8"""						) as vocab_handle:
         _lowerCAmelCase		: Union[str, Any]						    =  json.load(a__						)
       _lowerCAmelCase		: int						    =  {v: k for k, v in self.encoder.items()}
       with open(a__       ,  encoding="""utf-8"""						) as merges_handle:
         _lowerCAmelCase		: List[str]						    =  merges_handle.read().split("""\n"""						)[1:-1]
       _lowerCAmelCase		: List[Any]						    =  [tuple(merge.split()						) for merge in merges]
       _lowerCAmelCase		: List[Any]						    =  dict(zip(a__       ,  range(len(a__						)						)						)						)
       _lowerCAmelCase		: Optional[int]						    =  {}






     @property
     def 					__A						(	self						):
       return len(self.encoder						)






     def 					__A						(	self						):
       return dict(self.encoder       ,  **self.added_tokens_encoder						)






     def 					__A						(	self       ,  a__						):
       if token in self.cache:
         return self.cache[token]
       _lowerCAmelCase		: int						    =  tuple(a__						)
       _lowerCAmelCase		: Tuple						    =  tuple(list(word[:-1]						) + [word[-1] + """</w>"""]						)
       _lowerCAmelCase		: List[str]						    =  get_pairs(a__						)

       if not pairs:
         return token

       while True:
         _lowerCAmelCase		: List[str]						    =  min(a__       ,  key=lambda a__						: self.bpe_ranks.get(a__       ,  float("""inf"""						)						)						)
         if bigram not in self.bpe_ranks:
           break
         _lowerCAmelCase    ,			_lowerCAmelCase		: Optional[Any]						    =  bigram
         _lowerCAmelCase		: Optional[int]						    =  []
         _lowerCAmelCase		: str						    =  0
         while i < len(a__						):
           try:
             _lowerCAmelCase		: Dict						    =  word.index(a__       ,  a__						)
           except ValueError:
             new_word.extend(word[i:]						)
             break
           else:
             new_word.extend(word[i:j]						)
             _lowerCAmelCase		: Any						    =  j

           if word[i] == first and i < len(a__						) - 1 and word[i + 1] == second:
             new_word.append(first + second						)
             i += 2
           else:
             new_word.append(word[i]						)
             i += 1
         _lowerCAmelCase		: Dict						    =  tuple(a__						)
         _lowerCAmelCase		: List[Any]						    =  new_word
         if len(a__						) == 1:
           break
         else:
           _lowerCAmelCase		: Optional[int]						    =  get_pairs(a__						)
       _lowerCAmelCase		: Any						    =  """@@ """.join(a__						)
       _lowerCAmelCase		: List[Any]						    =  word[:-4]
       _lowerCAmelCase		: Optional[int]						    =  word
       return word






     def 					__A						(	self       ,  a__						):
       _lowerCAmelCase		: Optional[Any]						    =  []

       _lowerCAmelCase		: Optional[Any]						    =  re.findall(r"""\S+\n?"""       ,  a__						)

       for token in words:
         split_tokens.extend(list(self.bpe(a__						).split(""" """						)						)						)
       return split_tokens






     def 					__A						(	self       ,  a__						):
       return self.encoder.get(a__       ,  self.encoder.get(self.unk_token						)						)






     def 					__A						(	self       ,  a__						):
       return self.decoder.get(a__       ,  self.unk_token						)






     def 					__A						(	self       ,  a__						):
       _lowerCAmelCase		: List[str]						    =  """ """.join(a__						).replace("""@@ """       ,  """"""						).strip()
       return out_string






     def 					__A						(	self       ,  a__       ,  a__ = None						):
       if not os.path.isdir(a__						):
         logger.error(F"Vocabulary path ({save_directory}) should be a directory"						)
         return
       _lowerCAmelCase		: Union[str, Any]						    =  os.path.join(
           a__       ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
       _lowerCAmelCase		: Any						    =  os.path.join(
           a__       ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

       with open(a__       ,  """w"""       ,  encoding="""utf-8"""						) as f:
         f.write(json.dumps(self.encoder       ,  indent=2       ,  sort_keys=a__       ,  ensure_ascii=a__						) + """\n"""						)

       _lowerCAmelCase		: Optional[int]						    =  0
       with open(a__       ,  """w"""       ,  encoding="""utf-8"""						) as writer:
         writer.write("""#version: 0.2\n"""						)
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,  key=lambda a__						: kv[1]						):
           if index != token_index:
             logger.warning(
                 F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                 """ Please check that the tokenizer is not corrupted!"""						)
             _lowerCAmelCase		: List[Any]						    =  token_index
           writer.write(""" """.join(a__						) + """\n"""						)
           index += 1

       return vocab_file, merge_file

     # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
     #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
     #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
     #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
     #     return ''.join(tokens_generated_so_far)





"""simple docstring"""




import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		__A							(				SCREAMING_SNAKE_CASE_					,			unittest.TestCase	):
     _UpperCamelCase							:    Dict	      =	CodeGenTokenizer
     _UpperCamelCase							:    Dict	      =	CodeGenTokenizerFast
     _UpperCamelCase							:    Tuple	      =	True
     _UpperCamelCase							:    List[Any]	      =	{"add_prefix_space": True}
     _UpperCamelCase							:    str	      =	False






     def 					__A						(	self						):
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       _lowerCAmelCase		: Union[str, Any]						    =  [
           """l""",
           """o""",
           """w""",
           """e""",
           """r""",
           """s""",
           """t""",
           """i""",
           """d""",
           """n""",
           """\u0120""",
           """\u0120l""",
           """\u0120n""",
           """\u0120lo""",
           """\u0120low""",
           """er""",
           """\u0120lowest""",
           """\u0120newer""",
           """\u0120wider""",
           """<unk>""",
           """<|endoftext|>""",
       ]
       _lowerCAmelCase		: Optional[int]						    =  dict(zip(a__       ,  range(len(a__						)						)						)						)
       _lowerCAmelCase		: str						    =  ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
       _lowerCAmelCase		: Any						    =  {"""unk_token""": """<unk>"""}

       _lowerCAmelCase		: List[str]						    =  os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES["""vocab_file"""]						)
       _lowerCAmelCase		: List[Any]						    =  os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES["""merges_file"""]						)
       with open(self.vocab_file       ,  """w"""       ,  encoding="""utf-8"""						) as fp:
         fp.write(json.dumps(a__						) + """\n"""						)
       with open(self.merges_file       ,  """w"""       ,  encoding="""utf-8"""						) as fp:
         fp.write("""\n""".join(a__						)						)






     def 					__A						(	self       ,  **a__						):
       kwargs.update(self.special_tokens_map						)
       return CodeGenTokenizer.from_pretrained(self.tmpdirname       ,  **a__						)






     def 					__A						(	self       ,  **a__						):
       kwargs.update(self.special_tokens_map						)
       return CodeGenTokenizerFast.from_pretrained(self.tmpdirname       ,  **a__						)






     def 					__A						(	self       ,  a__						):
       _lowerCAmelCase		: str						    =  """lower newer"""
       _lowerCAmelCase		: Tuple						    =  """lower newer"""
       return input_text, output_text






     def 					__A						(	self						):
       _lowerCAmelCase		: str						    =  CodeGenTokenizer(self.vocab_file       ,  self.merges_file       ,  **self.special_tokens_map						)
       _lowerCAmelCase		: int						    =  """lower newer"""
       _lowerCAmelCase		: List[str]						    =  ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
       _lowerCAmelCase		: Optional[int]						    =  tokenizer.tokenize(a__       ,  add_prefix_space=a__						)
       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: Tuple						    =  tokens + [tokenizer.unk_token]
       _lowerCAmelCase		: List[str]						    =  [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(a__						)       ,  a__						)






     def 					__A						(	self						):
       if not self.test_rust_tokenizer:
         return

       _lowerCAmelCase		: Optional[int]						    =  self.get_tokenizer()
       _lowerCAmelCase		: Optional[int]						    =  self.get_rust_tokenizer(add_prefix_space=a__						)

       _lowerCAmelCase		: Any						    =  """lower newer"""

       # Testing tokenization
       _lowerCAmelCase		: Any						    =  tokenizer.tokenize(a__       ,  add_prefix_space=a__						)
       _lowerCAmelCase		: int						    =  rust_tokenizer.tokenize(a__						)
       self.assertListEqual(a__       ,  a__						)

       # Testing conversion to ids without special tokens
       _lowerCAmelCase		: Union[str, Any]						    =  tokenizer.encode(a__       ,  add_special_tokens=a__       ,  add_prefix_space=a__						)
       _lowerCAmelCase		: Dict						    =  rust_tokenizer.encode(a__       ,  add_special_tokens=a__						)
       self.assertListEqual(a__       ,  a__						)

       # Testing conversion to ids with special tokens
       _lowerCAmelCase		: int						    =  self.get_rust_tokenizer(add_prefix_space=a__						)
       _lowerCAmelCase		: Optional[int]						    =  tokenizer.encode(a__       ,  add_prefix_space=a__						)
       _lowerCAmelCase		: Any						    =  rust_tokenizer.encode(a__						)
       self.assertListEqual(a__       ,  a__						)

       # Testing the unknown token
       _lowerCAmelCase		: List[str]						    =  tokens + [rust_tokenizer.unk_token]
       _lowerCAmelCase		: Dict						    =  [14, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a__						)       ,  a__						)






     def 					__A						(	self       ,  *a__       ,  **a__						):
       # It's very difficult to mix/test pretokenization with byte-level
       # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
       pass






     def 					__A						(	self       ,  a__=15						):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
           _lowerCAmelCase		: List[Any]						    =  self.rust_tokenizer_class.from_pretrained(a__       ,  **a__						)

           # Simple input
           _lowerCAmelCase		: Dict						    =  """This is a simple input"""
           _lowerCAmelCase		: Optional[int]						    =  ["""This is a simple input 1""", """This is a simple input 2"""]
           _lowerCAmelCase		: Optional[int]						    =  ("""This is a simple input""", """This is a pair""")
           _lowerCAmelCase		: str						    =  [
               ("""This is a simple input 1""", """This is a simple input 2"""),
               ("""This is a simple pair 1""", """This is a simple pair 2"""),
           ]

           # Simple input tests
           self.assertRaises(a__       ,  tokenizer_r.encode       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Simple input
           self.assertRaises(a__       ,  tokenizer_r.encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Simple input
           self.assertRaises(
               a__       ,  tokenizer_r.batch_encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""       ,  )

           # Pair input
           self.assertRaises(a__       ,  tokenizer_r.encode       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Pair input
           self.assertRaises(a__       ,  tokenizer_r.encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Pair input
           self.assertRaises(
               a__       ,  tokenizer_r.batch_encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""       ,  )






     def 					__A						(	self						):
       _lowerCAmelCase		: Any						    =  CodeGenTokenizer.from_pretrained(self.tmpdirname       ,  pad_token="""<pad>"""						)

       # Simple input
       _lowerCAmelCase		: Union[str, Any]						    =  """This is a simple input"""
       _lowerCAmelCase		: Dict						    =  ["""This is a simple input looooooooong""", """This is a simple input"""]
       _lowerCAmelCase		: Any						    =  ("""This is a simple input""", """This is a pair""")
       _lowerCAmelCase		: Optional[int]						    =  [
           ("""This is a simple input loooooong""", """This is a simple input"""),
           ("""This is a simple pair loooooong""", """This is a simple pair"""),
       ]

       _lowerCAmelCase		: Optional[int]						    =  tokenizer.pad_token_id

       _lowerCAmelCase		: Any						    =  tokenizer(a__       ,  padding="""max_length"""       ,  max_length=30       ,  return_tensors="""np"""						)
       _lowerCAmelCase		: str						    =  tokenizer(a__       ,  padding=a__       ,  truncate=a__       ,  return_tensors="""np"""						)
       _lowerCAmelCase		: int						    =  tokenizer(*a__       ,  padding="""max_length"""       ,  max_length=60       ,  return_tensors="""np"""						)
       _lowerCAmelCase		: int						    =  tokenizer(a__       ,  padding=a__       ,  truncate=a__       ,  return_tensors="""np"""						)

       # s
       # test single string max_length padding
       self.assertEqual(out_s["""input_ids"""].shape[-1]       ,  30						)
       self.assertTrue(pad_token_id in out_s["""input_ids"""]						)
       self.assertTrue(0 in out_s["""attention_mask"""]						)

       # s2
       # test automatic padding
       self.assertEqual(out_sa["""input_ids"""].shape[-1]       ,  33						)
       # long slice doesn't have padding
       self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]						)
       self.assertFalse(0 in out_sa["""attention_mask"""][0]						)
       # short slice does have padding
       self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]						)
       self.assertTrue(0 in out_sa["""attention_mask"""][1]						)

       # p
       # test single pair max_length padding
       self.assertEqual(out_p["""input_ids"""].shape[-1]       ,  60						)
       self.assertTrue(pad_token_id in out_p["""input_ids"""]						)
       self.assertTrue(0 in out_p["""attention_mask"""]						)

       # p2
       # test automatic padding pair
       self.assertEqual(out_pa["""input_ids"""].shape[-1]       ,  52						)
       # long slice pair doesn't have padding
       self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]						)
       self.assertFalse(0 in out_pa["""attention_mask"""][0]						)
       # short slice pair does have padding
       self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]						)
       self.assertTrue(0 in out_pa["""attention_mask"""][1]						)






     def 					__A						(	self						):
       _lowerCAmelCase		: List[str]						    =  """$$$"""
       _lowerCAmelCase		: str						    =  CodeGenTokenizer.from_pretrained(self.tmpdirname       ,  bos_token=a__       ,  add_bos_token=a__						)

       _lowerCAmelCase		: Tuple						    =  """This is a simple input"""
       _lowerCAmelCase		: Union[str, Any]						    =  ["""This is a simple input 1""", """This is a simple input 2"""]

       _lowerCAmelCase		: List[str]						    =  tokenizer.bos_token_id

       _lowerCAmelCase		: str						    =  tokenizer(a__						)
       _lowerCAmelCase		: Union[str, Any]						    =  tokenizer(a__						)

       self.assertEqual(out_s.input_ids[0]       ,  a__						)
       self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

       _lowerCAmelCase		: Optional[Any]						    =  tokenizer.decode(out_s.input_ids						)
       _lowerCAmelCase		: Optional[int]						    =  tokenizer.batch_decode(out_sa.input_ids						)

       self.assertEqual(decode_s.split()[0]       ,  a__						)
       self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)






     @slow
     def 					__A						(	self						):
       _lowerCAmelCase		: int						    =  CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""						)

       _lowerCAmelCase		: Optional[int]						    =  """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
       _lowerCAmelCase		: List[Any]						    =  """\nif len_a > len_b:      result = a\nelse:      result = b"""

       _lowerCAmelCase		: Tuple						    =  tokenizer.encode(a__						)
       _lowerCAmelCase		: Optional[Any]						    =  ["""^#""", re.escape("""<|endoftext|>"""						), """^'''""", """^\"\"\"""", """\n\n\n"""]
       _lowerCAmelCase		: int						    =  tokenizer.decode(a__       ,  truncate_before_pattern=a__						)
       self.assertEqual(a__       ,  a__						)






     def 					__A						(	self						):
       pass




