


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


lowerCAmelCase							=						logging.get_logger(__name__)

lowerCAmelCase							=						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase							=						{
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

lowerCAmelCase							=						{
    """junnyu/roformer_chinese_small""": 1_536,
    """junnyu/roformer_chinese_base""": 1_536,
    """junnyu/roformer_chinese_char_small""": 512,
    """junnyu/roformer_chinese_char_base""": 512,
    """junnyu/roformer_small_discriminator""": 128,
    """junnyu/roformer_small_generator""": 128,
}


lowerCAmelCase							=						{
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}




class        lowerCamelCase      (     _UpperCamelCase   ):
	_lowerCAmelCase    :		Optional[Any]        =	VOCAB_FILES_NAMES
	_lowerCAmelCase    :		Union[str, Any]        =	PRETRAINED_VOCAB_FILES_MAP
	_lowerCAmelCase    :		int        =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowerCAmelCase    :		Dict        =	PRETRAINED_INIT_CONFIGURATION
	_lowerCAmelCase    :		Tuple        =	RoFormerTokenizer


	def __init__(						self							,    lowercase__=None							,    lowercase__=None							,    lowercase__=True							,    lowercase__="[UNK]"							,    lowercase__="[SEP]"							,    lowercase__="[PAD]"							,    lowercase__="[CLS]"							,    lowercase__="[MASK]"							,    lowercase__=True							,    lowercase__=None							,    **lowercase__							,    ):
							super().__init__(
							    lowercase__							,    tokenizer_file=lowercase__							,    do_lower_case=lowercase__							,    unk_token=lowercase__							,    sep_token=lowercase__							,    pad_token=lowercase__							,    cls_token=lowercase__							,    mask_token=lowercase__							,    tokenize_chinese_chars=lowercase__							,    strip_accents=lowercase__							,    **lowercase__							,    )

							__UpperCAmelCase :       str  						=  json.loads(self.backend_tokenizer.normalizer.__getstate__())
							if (
							    pre_tok_state.get('''lowercase'''							,    lowercase__) != do_lower_case
							    or pre_tok_state.get('''strip_accents'''							,    lowercase__) != strip_accents
							):
													__UpperCAmelCase :       Optional[int]  						=  getattr(lowercase__							,    pre_tok_state.pop('''type'''))
													__UpperCAmelCase :       Optional[int]  						=  do_lower_case
													__UpperCAmelCase :       Dict  						=  strip_accents
													__UpperCAmelCase :       str  						=  pre_tok_class(**lowercase__)

							__UpperCAmelCase :       Dict  						=  do_lower_case


	def __getstate__(						self):
							__UpperCAmelCase :       str  						=  self.__dict__.copy()
							__UpperCAmelCase :       Any  						=  BertPreTokenizer()
							return state


	def __setstate__(						self							,    lowercase__):
							__UpperCAmelCase :       Optional[Any]  						=  d
							__UpperCAmelCase :       Optional[int]  						=  self.__dict__['''_tokenizer'''].get_vocab()
							__UpperCAmelCase :       Optional[Any]  						=  PreTokenizer.custom(JiebaPreTokenizer(lowercase__))


	def    A(						self							,    lowercase__							,    lowercase__=None):
							__UpperCAmelCase :       Dict  						=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

							return output


	def    A(						self							,    lowercase__							,    lowercase__ = None):
							__UpperCAmelCase :       int  						=  [self.sep_token_id]
							__UpperCAmelCase :       List[Any]  						=  [self.cls_token_id]
							if token_ids_a is None:
													return len(cls + token_ids_a + sep) * [0]
							return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


	def    A(						self							,    lowercase__							,    lowercase__ = None):
							__UpperCAmelCase :       List[str]  						=  self._tokenizer.model.save(lowercase__							,    name=lowercase__)
							return tuple(lowercase__)


	def    A(						self							,    lowercase__							,    lowercase__=None							,    lowercase__=None							,    lowercase__=False							,    **lowercase__							,    ):
							__UpperCAmelCase :       Optional[int]  						=  BertPreTokenizer()
							return super().save_pretrained(lowercase__							,    lowercase__							,    lowercase__							,    lowercase__							,    **lowercase__)





from typing import TYPE_CHECKING

from ....utils import _LazyModule


lowerCAmelCase							=						{"""tokenization_tapex""": ["""TapexTokenizer"""]}


if TYPE_CHECKING:
				from .tokenization_tapex import TapexTokenizer


else:
				import sys

				lowerCAmelCase							=						_LazyModule(__name__, globals()["""__file__"""], _import_structure)

