



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


UpperCamelCase__ =						logging.get_logger(__name__)

UpperCamelCase__ =						{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCamelCase__ =						[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

UpperCamelCase__ =						{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
UpperCamelCase__ =						{F"""funnel-transformer/{name}""": 5_12 for name in _model_names}
UpperCamelCase__ =						{F"""funnel-transformer/{name}""": {"do_lower_case": True} for name in _model_names}
class 			_lowerCAmelCase   ( _UpperCAmelCase			):






			"""simple docstring"""





			lowercase__      :  Optional[int]      =			VOCAB_FILES_NAMES
			lowercase__      :  int      =			PRETRAINED_VOCAB_FILES_MAP
			lowercase__      :  List[Any]      =			PRETRAINED_INIT_CONFIGURATION
			lowercase__      :  Tuple      =			FunnelTokenizer
			lowercase__      :  Union[str, Any]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase__      :  int      =			2





			def __init__(				self	:       List[str]			,     lowercase	:       Optional[int]=None			,     lowercase	:       str=None			,     lowercase	:       Any=True			,     lowercase	:       Dict="<unk>"			,     lowercase	:       Optional[Any]="<sep>"			,     lowercase	:       Tuple="<pad>"			,     lowercase	:       Dict="<cls>"			,     lowercase	:       int="<mask>"			,     lowercase	:       int="<s>"			,     lowercase	:       str="</s>"			,     lowercase	:       Tuple=True			,     lowercase	:       Union[str, Any]=True			,     lowercase	:       List[Any]=None			,     lowercase	:       Optional[int]="##"			,     **lowercase	:       str			,     )						->					Union[str, Any]:
									"""simple docstring"""







									super().__init__(
									    lowercase			,     tokenizer_file=lowercase			,     do_lower_case=lowercase			,     unk_token=lowercase			,     sep_token=lowercase			,     pad_token=lowercase			,     cls_token=lowercase			,     mask_token=lowercase			,     bos_token=lowercase			,     eos_token=lowercase			,     clean_text=lowercase			,     tokenize_chinese_chars=lowercase			,     strip_accents=lowercase			,     wordpieces_prefix=lowercase			,     **lowercase			,     )

									__lowercase					=				json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
									if (
									    normalizer_state.get("""lowercase"""			,     lowercase   ) != do_lower_case
									    or normalizer_state.get("""strip_accents"""			,     lowercase   ) != strip_accents
									    or normalizer_state.get("""handle_chinese_chars"""			,     lowercase   ) != tokenize_chinese_chars
									):
															__lowercase					=				getattr(lowercase			,     normalizer_state.pop("""type"""   )   )
															__lowercase					=				do_lower_case
															__lowercase					=				strip_accents
															__lowercase					=				tokenize_chinese_chars
															__lowercase					=				normalizer_class(**lowercase   )

									__lowercase					=				do_lower_case





			def 		snake_case__			(				self	:       int			,     lowercase	:       Any			,     lowercase	:       Dict=None   )						->					Optional[int]:
									"""simple docstring"""







									__lowercase					=				[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

									return output





			def 		snake_case__			(				self	:       int			,     lowercase	:       List[int]			,     lowercase	:       Optional[List[int]] = None   )						->					List[int]:
									"""simple docstring"""







									__lowercase					=				[self.sep_token_id]
									__lowercase					=				[self.cls_token_id]
									if token_ids_a is None:
															return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0]
									return len(cls   ) * [self.cls_token_type_id] + len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







			def 		snake_case__			(				self	:       List[str]			,     lowercase	:       str			,     lowercase	:       Optional[str] = None   )						->					Tuple[str]:
									"""simple docstring"""







									__lowercase					=				self._tokenizer.model.save(lowercase			,     name=lowercase   )
									return tuple(lowercase   )










import random






def  UpperCAmelCase__    (    lowercase__						,       lowercase__						,       lowercase__ = False   ) ->      dict:
						__lowercase					=				{i: [] for i in range(lowercase__   )}

						# if probability is greater or equal than 1, then generate a complete graph
						if probability >= 1:
												return complete_graph(lowercase__   )
						# if probability is lower or equal than 0, then return a graph without edges
						if probability <= 0:
												return graph

						# for each couple of nodes, add an edge from u to v
						# if the number randomly generated is greater than probability probability
						for i in range(lowercase__   ):
												for j in range(i + 1						,       lowercase__   ):
																		if random.random() < probability:
																								graph[i].append(lowercase__   )
																								if not directed:
																														# if the graph is undirected, add an edge in from j to i, either
																														graph[j].append(lowercase__   )
						return graph






def  UpperCAmelCase__    (    lowercase__   ) ->      dict:
						return {
						    i: [j for j in range(lowercase__   ) if i != j] for i in range(lowercase__   )
						}


if __name__ == "__main__":
			import doctest

			doctest.testmod()





