



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


UpperCAmelCase__     = logging.get_logger(__name__)

UpperCAmelCase__     = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__     = {
    "vocab_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"
        ),
        "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt",
        "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt",
        "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "google/realm-cc-news-pretrained-embedder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"
        ),
        "google/realm-cc-news-pretrained-encoder": (
            "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-scorer": (
            "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"
        ),
        "google/realm-cc-news-pretrained-openqa": (
            "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-openqa": (
            "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-nq-reader": (
            "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-openqa": (
            "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"
        ),
        "google/realm-orqa-wq-reader": (
            "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase__     = {
    "google/realm-cc-news-pretrained-embedder": 512,
    "google/realm-cc-news-pretrained-encoder": 512,
    "google/realm-cc-news-pretrained-scorer": 512,
    "google/realm-cc-news-pretrained-openqa": 512,
    "google/realm-orqa-nq-openqa": 512,
    "google/realm-orqa-nq-reader": 512,
    "google/realm-orqa-wq-openqa": 512,
    "google/realm-orqa-wq-reader": 512,
}

UpperCAmelCase__     = {
    "google/realm-cc-news-pretrained-embedder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-encoder": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-scorer": {"do_lower_case": True},
    "google/realm-cc-news-pretrained-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-openqa": {"do_lower_case": True},
    "google/realm-orqa-nq-reader": {"do_lower_case": True},
    "google/realm-orqa-wq-openqa": {"do_lower_case": True},
    "google/realm-orqa-wq-reader": {"do_lower_case": True},
}







class     __lowerCAmelCase					(    A		):
			UpperCamelCase           =  VOCAB_FILES_NAMES
			UpperCamelCase           =  PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase           =  PRETRAINED_INIT_CONFIGURATION
			UpperCamelCase           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase           =  RealmTokenizer



			def __init__(		self     :						List[str]				,	A     :						Tuple=None				,	A     :						Optional[Any]=None				,	A     :						Tuple=True				,	A     :						str="[UNK]"				,	A     :						Dict="[SEP]"				,	A     :						Any="[PAD]"				,	A     :						Any="[CLS]"				,	A     :						Dict="[MASK]"				,	A     :						Tuple=True				,	A     :						List[str]=None				,	**A     :						Tuple				,	)     ->					Optional[Any]:



						"""simple docstring"""






						super().__init__(
						    A				,	tokenizer_file=A				,	do_lower_case=A				,	unk_token=A				,	sep_token=A				,	pad_token=A				,	cls_token=A				,	mask_token=A				,	tokenize_chinese_chars=A				,	strip_accents=A				,	**A				,	)

						_UpperCAmelCase	 =     json.loads(self.backend_tokenizer.normalizer.__getstate__())
						if (
						    normalizer_state.get('lowercase'				,	A) != do_lower_case
						    or normalizer_state.get('strip_accents'				,	A) != strip_accents
						    or normalizer_state.get('handle_chinese_chars'				,	A) != tokenize_chinese_chars
						):
									_UpperCAmelCase	 =     getattr(A				,	normalizer_state.pop('type'))
									_UpperCAmelCase	 =     do_lower_case
									_UpperCAmelCase	 =     strip_accents
									_UpperCAmelCase	 =     tokenize_chinese_chars
									_UpperCAmelCase	 =     normalizer_class(**A)

						_UpperCAmelCase	 =     do_lower_case



			def 					_lowerCamelCase (		self     :						List[str]				,	A     :						Union[str, Any]				,	**A     :						Dict)     ->					Dict:



						"""simple docstring"""






						_UpperCAmelCase	 =     PaddingStrategy.MAX_LENGTH

						_UpperCAmelCase	 =     text
						_UpperCAmelCase	 =     kwargs.pop('text_pair'				,	A)
						_UpperCAmelCase	 =     kwargs.pop('return_tensors'				,	A)

						_UpperCAmelCase	 =     {
						    'input_ids': [],
						    'attention_mask': [],
						    'token_type_ids': [],
						}

						for idx, candidate_text in enumerate(A):
									if batch_text_pair is not None:
												_UpperCAmelCase	 =     batch_text_pair[idx]
									else:
												_UpperCAmelCase	 =     None

									_UpperCAmelCase	 =     super().__call__(A				,	A				,	return_tensors=A				,	**A)

									_UpperCAmelCase	 =     encoded_candidates.get('input_ids')
									_UpperCAmelCase	 =     encoded_candidates.get('attention_mask')
									_UpperCAmelCase	 =     encoded_candidates.get('token_type_ids')

									if encoded_input_ids is not None:
												output_data["input_ids"].append(A)
									if encoded_attention_mask is not None:
												output_data["attention_mask"].append(A)
									if encoded_token_type_ids is not None:
												output_data["token_type_ids"].append(A)

						_UpperCAmelCase	 =     {key: item for key, item in output_data.items() if len(A) != 0}

						return BatchEncoding(A				,	tensor_type=A)



			def 					_lowerCamelCase (		self     :						Dict				,	A     :						Dict				,	A     :						Optional[Any]=None)     ->					List[str]:



						"""simple docstring"""






						_UpperCAmelCase	 =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

						return output



			def 					_lowerCamelCase (		self     :						Optional[Any]				,	A     :						List[int]				,	A     :						Optional[List[int]] = None)     ->					List[int]:



						"""simple docstring"""






						_UpperCAmelCase	 =     [self.sep_token_id]
						_UpperCAmelCase	 =     [self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep) * [0]
						return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



			def 					_lowerCamelCase (		self     :						List[str]				,	A     :						str				,	A     :						Optional[str] = None)     ->					Tuple[str]:



						"""simple docstring"""






						_UpperCAmelCase	 =     self._tokenizer.model.save(A				,	name=A)
						return tuple(A)





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCAmelCase__     = {
    "configuration_ctrl": ["CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP", "CTRLConfig"],
    "tokenization_ctrl": ["CTRLTokenizer"],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__     = [
							    "CTRL_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "CTRLForSequenceClassification",
							    "CTRLLMHeadModel",
							    "CTRLModel",
							    "CTRLPreTrainedModel",
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__     = [
							    "TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "TFCTRLForSequenceClassification",
							    "TFCTRLLMHeadModel",
							    "TFCTRLModel",
							    "TFCTRLPreTrainedModel",
							]


if TYPE_CHECKING:
							from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
							from .tokenization_ctrl import CTRLTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_ctrl import (
														    CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
														    CTRLForSequenceClassification,
														    CTRLLMHeadModel,
														    CTRLModel,
														    CTRLPreTrainedModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_ctrl import (
														    TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFCTRLForSequenceClassification,
														    TFCTRLLMHeadModel,
														    TFCTRLModel,
														    TFCTRLPreTrainedModel,
														)

else:
							import sys

							UpperCAmelCase__     = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
