



"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


__snake_case							:     int   =					logging.get_logger(__name__)

__snake_case							:     List[Any]   =					{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__snake_case							:     Dict   =					[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

__snake_case							:     Optional[int]   =					{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
__snake_case							:     int   =					{F"""funnel-transformer/{name}""": 512 for name in _model_names}
__snake_case							:     Dict   =					{F"""funnel-transformer/{name}""": {'do_lower_case': True} for name in _model_names}



class     A__     (						SCREAMING_SNAKE_CASE_							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE					   =			VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE					   =			PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE					   =			PRETRAINED_INIT_CONFIGURATION
	SCREAMING_SNAKE_CASE					   =			FunnelTokenizer
	SCREAMING_SNAKE_CASE					   =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE					   =			2

	def __init__(  self:	List[Any] ,    _SCREAMING_SNAKE_CASE:	Any=None ,    _SCREAMING_SNAKE_CASE:	int=None ,    _SCREAMING_SNAKE_CASE:	Optional[Any]=True ,    _SCREAMING_SNAKE_CASE:	Union[str, Any]="<unk>" ,    _SCREAMING_SNAKE_CASE:	int="<sep>" ,    _SCREAMING_SNAKE_CASE:	Optional[Any]="<pad>" ,    _SCREAMING_SNAKE_CASE:	Dict="<cls>" ,    _SCREAMING_SNAKE_CASE:	Union[str, Any]="<mask>" ,    _SCREAMING_SNAKE_CASE:	List[str]="<s>" ,    _SCREAMING_SNAKE_CASE:	List[str]="</s>" ,    _SCREAMING_SNAKE_CASE:	List[Any]=True ,    _SCREAMING_SNAKE_CASE:	List[str]=True ,    _SCREAMING_SNAKE_CASE:	Union[str, Any]=None ,    _SCREAMING_SNAKE_CASE:	Optional[int]="##" ,    **_SCREAMING_SNAKE_CASE:	List[Any] ,    )       -> Optional[Any]:

				"""simple docstring"""

				super().__init__(
				    UpperCamelCase__ ,    tokenizer_file=UpperCamelCase__ ,    do_lower_case=UpperCamelCase__ ,    unk_token=UpperCamelCase__ ,    sep_token=UpperCamelCase__ ,    pad_token=UpperCamelCase__ ,    cls_token=UpperCamelCase__ ,    mask_token=UpperCamelCase__ ,    bos_token=UpperCamelCase__ ,    eos_token=UpperCamelCase__ ,    clean_text=UpperCamelCase__ ,    tokenize_chinese_chars=UpperCamelCase__ ,    strip_accents=UpperCamelCase__ ,    wordpieces_prefix=UpperCamelCase__ ,    **UpperCamelCase__ ,    )

				__lowerCAmelCase   :						Dict				= json.loads(self.backend_tokenizer.normalizer.__getstate__())
				if (
				    normalizer_state.get("lowercase" ,    UpperCamelCase__) != do_lower_case
				    or normalizer_state.get("strip_accents" ,    UpperCamelCase__) != strip_accents
				    or normalizer_state.get("handle_chinese_chars" ,    UpperCamelCase__) != tokenize_chinese_chars
				):
							__lowerCAmelCase   :						Tuple				= getattr(UpperCamelCase__ ,    normalizer_state.pop("type"))
							__lowerCAmelCase   :						int				= do_lower_case
							__lowerCAmelCase   :						Tuple				= strip_accents
							__lowerCAmelCase   :						Any				= tokenize_chinese_chars
							__lowerCAmelCase   :						Dict				= normalizer_class(**UpperCamelCase__)

				__lowerCAmelCase   :						Union[str, Any]				= do_lower_case

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict ,    _SCREAMING_SNAKE_CASE:	Union[str, Any] ,    _SCREAMING_SNAKE_CASE:	Tuple=None)       -> int:

				"""simple docstring"""

				__lowerCAmelCase   :						List[str]				= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[int] ,    _SCREAMING_SNAKE_CASE:	Dict ,    _SCREAMING_SNAKE_CASE:	Optional[int] = None)       -> Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						int				= [self.sep_token_id]
				__lowerCAmelCase   :						Optional[int]				= [self.cls_token_id]
				if token_ids_a is None:
							return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
				return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]







	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any] ,    _SCREAMING_SNAKE_CASE:	Tuple ,    _SCREAMING_SNAKE_CASE:	int = None)       -> str:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= self._tokenizer.model.save(UpperCamelCase__ ,    name=UpperCamelCase__)
				return tuple(UpperCamelCase__)




"""simple docstring"""






def     _lowercase       (						__snake_case      ,__snake_case			)		->     list:
			__lowerCAmelCase   :						List[str]				= len(__snake_case			)
			__lowerCAmelCase   :						Dict				= []
			for i in range(len(__snake_case			) - pat_len + 1			):
						__lowerCAmelCase   :						List[Any]				= True
						for j in range(__snake_case			):
									if s[i + j] != pattern[j]:
												__lowerCAmelCase   :						Union[str, Any]				= False
												break
						if match_found:
									position.append(__snake_case			)
			return position


if __name__ == "__main__":
	assert naive_pattern_search('ABCDEFG', 'DE') == [3]
	print(naive_pattern_search('ABAAABCDBBABCDDEBCABC', 'ABC'))