"""simple docstring"""






import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__						:						Optional[Any]		  =			logging.get_logger(__name__)

lowerCAmelCase__						:						str		  =			{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowerCAmelCase__						:						Tuple		  =			{
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

lowerCAmelCase__						:						Optional[int]		  =			{
    """ctrl""": 256,
}

lowerCAmelCase__						:						Any		  =			{
    """Pregnancy""": 168_629,
    """Christianity""": 7_675,
    """Explain""": 106_423,
    """Fitness""": 63_440,
    """Saving""": 63_163,
    """Ask""": 27_171,
    """Ass""": 95_985,
    """Joke""": 163_509,
    """Questions""": 45_622,
    """Thoughts""": 49_605,
    """Retail""": 52_342,
    """Feminism""": 164_338,
    """Writing""": 11_992,
    """Atheism""": 192_263,
    """Netflix""": 48_616,
    """Computing""": 39_639,
    """Opinion""": 43_213,
    """Alone""": 44_967,
    """Funny""": 58_917,
    """Gaming""": 40_358,
    """Human""": 4_088,
    """India""": 1_331,
    """Joker""": 77_138,
    """Diet""": 36_206,
    """Legal""": 11_859,
    """Norman""": 4_939,
    """Tip""": 72_689,
    """Weight""": 52_343,
    """Movies""": 46_273,
    """Running""": 23_425,
    """Science""": 2_090,
    """Horror""": 37_793,
    """Confession""": 60_572,
    """Finance""": 12_250,
    """Politics""": 16_360,
    """Scary""": 191_985,
    """Support""": 12_654,
    """Technologies""": 32_516,
    """Teenage""": 66_160,
    """Event""": 32_769,
    """Learned""": 67_460,
    """Notion""": 182_770,
    """Wikipedia""": 37_583,
    """Books""": 6_665,
    """Extract""": 76_050,
    """Confessions""": 102_701,
    """Conspiracy""": 75_932,
    """Links""": 63_674,
    """Narcissus""": 150_425,
    """Relationship""": 54_766,
    """Relationships""": 134_796,
    """Reviews""": 41_671,
    """News""": 4_256,
    """Translation""": 26_820,
    """multilingual""": 128_406,
}



def 						a_		(      lowerCamelCase ):
    UpperCAmelCase__    =							set()
    UpperCAmelCase__    =							word[0]
    for char in word[1:]:
        pairs.add((prev_char, char) )
        UpperCAmelCase__    =							char

    UpperCAmelCase__    =							set(lowerCamelCase )
    return pairs







class        snake_case (				_snake_case       ):




     """simple docstring"""

     snake_case__       					=				VOCAB_FILES_NAMES
     snake_case__       					=				PRETRAINED_VOCAB_FILES_MAP
     snake_case__       					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     snake_case__       					=				CONTROL_CODES







     def __init__(       self     :			Optional[Any]					,lowerCamelCase__     :			Optional[Any]					,lowerCamelCase__     :			int					,lowerCamelCase__     :			List[Any]="<unk>"					,**lowerCamelCase__     :			int    ):
         super().__init__(unk_token=snake_case_					,**snake_case_    )

         with open(snake_case_					,encoding='utf-8'    ) as vocab_handle:
             UpperCAmelCase__    =							json.load(snake_case_    )
         UpperCAmelCase__    =							{v: k for k, v in self.encoder.items()}
         with open(snake_case_					,encoding='utf-8'    ) as merges_handle:
             UpperCAmelCase__    =							merges_handle.read().split('\n'    )[1:-1]
         UpperCAmelCase__    =							[tuple(merge.split()    ) for merge in merges]
         UpperCAmelCase__    =							dict(zip(snake_case_					,range(len(snake_case_    )    )    )    )
         UpperCAmelCase__    =							{}







     @property
     def 	__lowerCAmelCase		(       self     :			Union[str, Any]    ):
         return len(self.encoder    )







     def 	__lowerCAmelCase		(       self     :			str    ):
         return dict(self.encoder					,**self.added_tokens_encoder    )







     def 	__lowerCAmelCase		(       self     :			str					,lowerCamelCase__     :			Any    ):
         if token in self.cache:
             return self.cache[token]
         UpperCAmelCase__    =							tuple(snake_case_    )
         UpperCAmelCase__    =							tuple(list(word[:-1]    ) + [word[-1] + '</w>']    )
         UpperCAmelCase__    =							get_pairs(snake_case_    )

         if not pairs:
             return token

         while True:
             UpperCAmelCase__    =							min(snake_case_					,key=lambda lowerCamelCase__    : self.bpe_ranks.get(snake_case_					,float('inf'    )    )    )
             if bigram not in self.bpe_ranks:
                 break
             UpperCAmelCase__    =							bigram
             UpperCAmelCase__    =							[]
             UpperCAmelCase__    =							0
             while i < len(snake_case_    ):
                 try:
                     UpperCAmelCase__    =							word.index(snake_case_					,snake_case_    )
                 except ValueError:
                     new_word.extend(word[i:]    )
                     break
                 else:
                     new_word.extend(word[i:j]    )
                     UpperCAmelCase__    =							j

                 if word[i] == first and i < len(snake_case_    ) - 1 and word[i + 1] == second:
                     new_word.append(first + second    )
                     i += 2
                 else:
                     new_word.append(word[i]    )
                     i += 1
             UpperCAmelCase__    =							tuple(snake_case_    )
             UpperCAmelCase__    =							new_word
             if len(snake_case_    ) == 1:
                 break
             else:
                 UpperCAmelCase__    =							get_pairs(snake_case_    )
         UpperCAmelCase__    =							"@@ ".join(snake_case_    )
         UpperCAmelCase__    =							word[:-4]
         UpperCAmelCase__    =							word
         return word







     def 	__lowerCAmelCase		(       self     :			List[str]					,lowerCamelCase__     :			Optional[int]    ):
         UpperCAmelCase__    =							[]

         UpperCAmelCase__    =							re.findall(R'\S+\n?'					,snake_case_    )

         for token in words:
             split_tokens.extend(list(self.bpe(snake_case_    ).split(' '    )    )    )
         return split_tokens







     def 	__lowerCAmelCase		(       self     :			int					,lowerCamelCase__     :			int    ):
         return self.encoder.get(snake_case_					,self.encoder.get(self.unk_token    )    )







     def 	__lowerCAmelCase		(       self     :			int					,lowerCamelCase__     :			int    ):
         return self.decoder.get(snake_case_					,self.unk_token    )







     def 	__lowerCAmelCase		(       self     :			Dict					,lowerCamelCase__     :			Optional[int]    ):
         UpperCAmelCase__    =							" ".join(snake_case_    ).replace('@@ '					,''    ).strip()
         return out_string







     def 	__lowerCAmelCase		(       self     :			Optional[int]					,lowerCamelCase__     :			List[Any]					,lowerCamelCase__     :			Optional[Any] = None    ):
         if not os.path.isdir(snake_case_    ):
             logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
             return
         UpperCAmelCase__    =							os.path.join(
             snake_case_					,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
         UpperCAmelCase__    =							os.path.join(
             snake_case_					,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']    )

         with open(snake_case_					,'w'					,encoding='utf-8'    ) as f:
             f.write(json.dumps(self.encoder					,indent=2					,sort_keys=snake_case_					,ensure_ascii=snake_case_    ) + '\n'    )

         UpperCAmelCase__    =							0
         with open(snake_case_					,'w'					,encoding='utf-8'    ) as writer:
             writer.write('#version: 0.2\n'    )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,key=lambda lowerCamelCase__    : kv[1]    ):
                 if index != token_index:
                     logger.warning(
                         f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                         ' Please check that the tokenizer is not corrupted!'    )
                     UpperCAmelCase__    =							token_index
                 writer.write(' '.join(snake_case_    ) + '\n'    )
                 index += 1

         return vocab_file, merge_file

     # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
     #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
     #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
     #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
     #     return ''.join(tokens_generated_so_far)




"""simple docstring"""






import os
import sys
from contextlib import contextmanager


# Windows only
if os.name == "nt":
						import ctypes
						import msvcrt  # noqa
						class        snake_case (				ctypes.Structure       ):




											"""simple docstring"""

											snake_case__       					=				[("size", ctypes.c_int), ("visible", ctypes.c_byte)]



def 						a_		(      ):
				if os.name == "nt":
								UpperCAmelCase__    =							CursorInfo()
								UpperCAmelCase__    =							ctypes.windll.kernelaa.GetStdHandle(-1_1 )
								ctypes.windll.kernelaa.GetConsoleCursorInfo(lowerCamelCase      ,    ctypes.byref(lowerCamelCase ) )
								UpperCAmelCase__    =							False
								ctypes.windll.kernelaa.SetConsoleCursorInfo(lowerCamelCase      ,    ctypes.byref(lowerCamelCase ) )
				elif os.name == "posix":
								sys.stdout.write('\033[?25l' )
								sys.stdout.flush()



def 						a_		(      ):
				if os.name == "nt":
								UpperCAmelCase__    =							CursorInfo()
								UpperCAmelCase__    =							ctypes.windll.kernelaa.GetStdHandle(-1_1 )
								ctypes.windll.kernelaa.GetConsoleCursorInfo(lowerCamelCase      ,    ctypes.byref(lowerCamelCase ) )
								UpperCAmelCase__    =							True
								ctypes.windll.kernelaa.SetConsoleCursorInfo(lowerCamelCase      ,    ctypes.byref(lowerCamelCase ) )
				elif os.name == "posix":
								sys.stdout.write('\033[?25h' )
								sys.stdout.flush()







@contextmanager
def 						a_		(      ):
				try:
								hide_cursor()
								yield
				finally:
								show_cursor()



