

'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


__lowerCamelCase		       =    {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__lowerCamelCase		       =    {
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

__lowerCamelCase		       =    {
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

__lowerCamelCase		       =    {
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}


class 					A__      (      _UpperCAmelCase			):
    lowercase						=		VOCAB_FILES_NAMES
    lowercase						=		PRETRAINED_VOCAB_FILES_MAP
    lowercase						=		PRETRAINED_INIT_CONFIGURATION
    lowercase						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase						=		ElectraTokenizer



    def __init__(					self		,				UpperCamelCase__=None		,				UpperCamelCase__=None		,				UpperCamelCase__=True		,				UpperCamelCase__="[UNK]"		,				UpperCamelCase__="[SEP]"		,				UpperCamelCase__="[PAD]"		,				UpperCamelCase__="[CLS]"		,				UpperCamelCase__="[MASK]"		,				UpperCamelCase__=True		,				UpperCamelCase__=None		,				**UpperCamelCase__		,				)						->      Dict:





           '''simple docstring'''
           super().__init__(
               lowerCamelCase_		,				tokenizer_file=lowerCamelCase_		,				do_lower_case=lowerCamelCase_		,				unk_token=lowerCamelCase_		,				sep_token=lowerCamelCase_		,				pad_token=lowerCamelCase_		,				cls_token=lowerCamelCase_		,				mask_token=lowerCamelCase_		,				tokenize_chinese_chars=lowerCamelCase_		,				strip_accents=lowerCamelCase_		,				**lowerCamelCase_		,				)

           A_						=      json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
           if (
               normalizer_state.get("""lowercase"""		,				lowerCamelCase_       ) != do_lower_case
               or normalizer_state.get("""strip_accents"""		,				lowerCamelCase_       ) != strip_accents
               or normalizer_state.get("""handle_chinese_chars"""		,				lowerCamelCase_       ) != tokenize_chinese_chars
           ):
                  A_						=      getattr(lowerCamelCase_		,				normalizer_state.pop("""type"""       )       )
                  A_						=      do_lower_case
                  A_						=      strip_accents
                  A_						=      tokenize_chinese_chars
                  A_						=      normalizer_class(**lowerCamelCase_       )

           A_						=      do_lower_case



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__=None       )						->      str:





           '''simple docstring'''
           A_						=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__ = None       )						->      List[int]:





           '''simple docstring'''
           A_						=      [self.sep_token_id]
           A_						=      [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep       ) * [0]
           return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__ = None       )						->      Tuple[str]:





           '''simple docstring'''
           A_						=      self._tokenizer.model.save(lowerCamelCase_		,				name=lowerCamelCase_       )
           return tuple(lowerCamelCase_       )



'''simple docstring'''







def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__				)			->			float:
       _validate_point(UpperCAmelCase__				)
       _validate_point(UpperCAmelCase__				)
       if len(UpperCAmelCase__				) != len(UpperCAmelCase__				):
              raise ValueError("""Both points must be in the same n-dimensional space"""				)

       return float(sum(abs(a - b				) for a, b in zip(UpperCAmelCase__,       UpperCAmelCase__				)				)				)
def       UpperCAmelCase__    (      UpperCAmelCase__				)			->			None:
       if point:
              if isinstance(UpperCAmelCase__,       UpperCAmelCase__				):
                     for item in point:
                            if not isinstance(UpperCAmelCase__,       (int, float)				):
                                   A_						=      (
                                       """Expected a list of numbers as input, found """
                                       F'''{type(UpperCAmelCase__				).__name__}'''
                                   )
                                   raise TypeError(UpperCAmelCase__				)
              else:
                     A_						=      F'''Expected a list of numbers as input, found {type(UpperCAmelCase__				).__name__}'''
                     raise TypeError(UpperCAmelCase__				)
       else:
              raise ValueError("""Missing an input"""				)
def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__				)			->			float:
       _validate_point(UpperCAmelCase__				)
       _validate_point(UpperCAmelCase__				)
       if len(UpperCAmelCase__				) != len(UpperCAmelCase__				):
              raise ValueError("""Both points must be in the same n-dimensional space"""				)

       return float(sum(abs(x - y				) for x, y in zip(UpperCAmelCase__,       UpperCAmelCase__				)				)				)


if __name__ == "__main__":
      import doctest

      doctest.testmod()
