



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


a_			=					logging.get_logger(__name__)

a_			=					{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

a_			=					{
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

a_			=					{
    "junnyu/roformer_chinese_small": 1_536,
    "junnyu/roformer_chinese_base": 1_536,
    "junnyu/roformer_chinese_char_small": 512,
    "junnyu/roformer_chinese_char_base": 512,
    "junnyu/roformer_small_discriminator": 128,
    "junnyu/roformer_small_generator": 128,
}


a_			=					{
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}



class  __lowerCAmelCase     (	__lowerCAmelCase   ):
 lowerCAmelCase__       					=  VOCAB_FILES_NAMES
 lowerCAmelCase__       					=  PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       					=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__       					=  PRETRAINED_INIT_CONFIGURATION
 lowerCAmelCase__       					=  RoFormerTokenizer




 def __init__(						self	,				__UpperCAmelCase=None	,				__UpperCAmelCase=None	,				__UpperCAmelCase=True	,				__UpperCAmelCase="[UNK]"	,				__UpperCAmelCase="[SEP]"	,				__UpperCAmelCase="[PAD]"	,				__UpperCAmelCase="[CLS]"	,				__UpperCAmelCase="[MASK]"	,				__UpperCAmelCase=True	,				__UpperCAmelCase=None	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    super().__init__(
        lowerCamelCase__	,				tokenizer_file=lowerCamelCase__	,				do_lower_case=lowerCamelCase__	,				unk_token=lowerCamelCase__	,				sep_token=lowerCamelCase__	,				pad_token=lowerCamelCase__	,				cls_token=lowerCamelCase__	,				mask_token=lowerCamelCase__	,				tokenize_chinese_chars=lowerCamelCase__	,				strip_accents=lowerCamelCase__	,				**lowerCamelCase__	,				)

    __lowerCamelCase													=    json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
    if (
        pre_tok_state.get('''lowercase'''	,				lowerCamelCase__				) != do_lower_case
        or pre_tok_state.get('''strip_accents'''	,				lowerCamelCase__				) != strip_accents
    ):
       __lowerCamelCase													=    getattr(lowerCamelCase__	,				pre_tok_state.pop('''type'''				)				)
       __lowerCamelCase													=    do_lower_case
       __lowerCamelCase													=    strip_accents
       __lowerCamelCase													=    pre_tok_class(**lowerCamelCase__				)

    __lowerCamelCase													=    do_lower_case




 def __getstate__(						self				):



    '''simple docstring'''
    __lowerCamelCase													=    self.__dict__.copy()
    __lowerCamelCase													=    BertPreTokenizer()
    return state




 def __setstate__(						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    __lowerCamelCase													=    d
    __lowerCamelCase													=    self.__dict__['''_tokenizer'''].get_vocab()
    __lowerCamelCase													=    PreTokenizer.custom(JiebaPreTokenizer(lowerCamelCase__				)				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase=None				):



    '''simple docstring'''
    __lowerCamelCase													=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

    return output




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    __lowerCamelCase													=    [self.sep_token_id]
    __lowerCamelCase													=    [self.cls_token_id]
    if token_ids_a is None:
       return len(cls + token_ids_a + sep				) * [0]
    return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = None				):



    '''simple docstring'''
    __lowerCamelCase													=    self._tokenizer.model.save(lowerCamelCase__	,				name=lowerCamelCase__				)
    return tuple(lowerCamelCase__				)




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase=None	,				__UpperCAmelCase=None	,				__UpperCAmelCase=False	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    __lowerCamelCase													=    BertPreTokenizer()
    return super().save_pretrained(lowerCamelCase__	,				lowerCamelCase__	,				lowerCamelCase__	,				lowerCamelCase__	,				**lowerCamelCase__				)









def 		a__			(		_UpperCamelCase			: str ,_UpperCamelCase			: str = " "							):
   __lowerCamelCase													=    []

   __lowerCamelCase													=    0
   for index, char in enumerate(_UpperCamelCase							):
      if char == separator:
         split_words.append(string[last_index:index]							)
         __lowerCamelCase													=    index + 1
      elif index + 1 == len(_UpperCamelCase							):
         split_words.append(string[last_index : index + 1]							)
   return split_words


if __name__ == "__main__":
  from doctest import testmod

  testmod()






