



import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case	=						logging.get_logger(__name__)

_snake_case	=						{
    "vocab_file": "vocab.txt",
    "merges_file": "bpe.codes",
}

_snake_case	=						{
    "vocab_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt",
    },
    "merges_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes",
    },
}

_snake_case	=						{
    "vinai/phobert-base": 256,
    "vinai/phobert-large": 256,
}






def        A       (      _lowerCamelCase						):





							'''simple docstring'''




							_lowerCAmelCase  :							Any				     =					set()
							_lowerCAmelCase  :							Optional[int]				     =					word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)						)
														_lowerCAmelCase  :							Optional[int]				     =					char

							_lowerCAmelCase  :							Dict				     =					set(_lowerCamelCase						)
							return pairs



class 						UpperCAmelCase_					( a):
							lowerCamelCase__         =	VOCAB_FILES_NAMES
							lowerCamelCase__         =	PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase__         =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







							def __init__(     self,    __a,    __a,    __a="<s>",    __a="</s>",    __a="</s>",    __a="<s>",    __a="<unk>",    __a="<pad>",    __a="<mask>",    **__a,    ):

														'''simple docstring'''

														super().__init__(
														    bos_token=__a,    eos_token=__a,    unk_token=__a,    sep_token=__a,    cls_token=__a,    pad_token=__a,    mask_token=__a,    **__a,    )

														_lowerCAmelCase  :							str				     =					vocab_file
														_lowerCAmelCase  :							Dict				     =					merges_file

														_lowerCAmelCase  :							Optional[Any]				     =					{}
														_lowerCAmelCase  :							Tuple				     =					0
														_lowerCAmelCase  :							Optional[int]				     =					1
														_lowerCAmelCase  :							Dict				     =					2
														_lowerCAmelCase  :							Optional[int]				     =					3

														self.add_from_file(__a)

														_lowerCAmelCase  :							List[Any]				     =					{v: k for k, v in self.encoder.items()}

														with open(__a,    encoding="utf-8") as merges_handle:
																					_lowerCAmelCase  :							Union[str, Any]				     =					merges_handle.read().split("\n")[:-1]
														_lowerCAmelCase  :							Any				     =					[tuple(merge.split()[:-1]) for merge in merges]
														_lowerCAmelCase  :							Optional[Any]				     =					dict(zip(__a,    range(len(__a))))
														_lowerCAmelCase  :							Optional[Any]				     =					{}







							def      snake_case__   (     self,    __a,    __a = None):

														'''simple docstring'''

														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														_lowerCAmelCase  :							Any				     =					[self.cls_token_id]
														_lowerCAmelCase  :							Optional[int]				     =					[self.sep_token_id]
														return cls + token_ids_a + sep + sep + token_ids_a + sep







							def      snake_case__   (     self,    __a,    __a = None,    __a = False):

														'''simple docstring'''

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=__a,    token_ids_a=__a,    already_has_special_tokens=__a)

														if token_ids_a is None:
																					return [1] + ([0] * len(__a)) + [1]
														return [1] + ([0] * len(__a)) + [1, 1] + ([0] * len(__a)) + [1]







							def      snake_case__   (     self,    __a,    __a = None):

														'''simple docstring'''

														_lowerCAmelCase  :							str				     =					[self.sep_token_id]
														_lowerCAmelCase  :							Optional[Any]				     =					[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]







							@property
							def      snake_case__   (     self):

														'''simple docstring'''

														return len(self.encoder)







							def      snake_case__   (     self):

														'''simple docstring'''

														return dict(self.encoder,    **self.added_tokens_encoder)







							def      snake_case__   (     self,    __a):

														'''simple docstring'''

														if token in self.cache:
																					return self.cache[token]
														_lowerCAmelCase  :							List[Any]				     =					tuple(__a)
														_lowerCAmelCase  :							Optional[Any]				     =					tuple(list(word[:-1]) + [word[-1] + "</w>"])
														_lowerCAmelCase  :							List[str]				     =					get_pairs(__a)

														if not pairs:
																					return token

														while True:
																					_lowerCAmelCase  :							Optional[Any]				     =					min(__a,    key=lambda __a: self.bpe_ranks.get(__a,    float("inf")))
																					if bigram not in self.bpe_ranks:
																												break
																					_lowerCAmelCase     ,					_lowerCAmelCase  :							Tuple				     =					bigram
																					_lowerCAmelCase  :							List[str]				     =					[]
																					_lowerCAmelCase  :							List[Any]				     =					0
																					while i < len(__a):
																												try:
																																			_lowerCAmelCase  :							Tuple				     =					word.index(__a,    __a)
																												except ValueError:
																																			new_word.extend(word[i:])
																																			break
																												else:
																																			new_word.extend(word[i:j])
																																			_lowerCAmelCase  :							List[str]				     =					j

																												if word[i] == first and i < len(__a) - 1 and word[i + 1] == second:
																																			new_word.append(first + second)
																																			i += 2
																												else:
																																			new_word.append(word[i])
																																			i += 1
																					_lowerCAmelCase  :							int				     =					tuple(__a)
																					_lowerCAmelCase  :							Dict				     =					new_word
																					if len(__a) == 1:
																												break
																					else:
																												_lowerCAmelCase  :							Optional[int]				     =					get_pairs(__a)
														_lowerCAmelCase  :							str				     =					"@@ ".join(__a)
														_lowerCAmelCase  :							Union[str, Any]				     =					word[:-4]
														_lowerCAmelCase  :							int				     =					word
														return word







							def      snake_case__   (     self,    __a):

														'''simple docstring'''

														_lowerCAmelCase  :							Optional[int]				     =					[]

														_lowerCAmelCase  :							List[str]				     =					re.findall(R"\S+\n?",    __a)

														for token in words:
																					split_tokens.extend(list(self.bpe(__a).split(" ")))
														return split_tokens







							def      snake_case__   (     self,    __a):

														'''simple docstring'''

														return self.encoder.get(__a,    self.encoder.get(self.unk_token))







							def      snake_case__   (     self,    __a):

														'''simple docstring'''

														return self.decoder.get(__a,    self.unk_token)







							def      snake_case__   (     self,    __a):

														'''simple docstring'''

														_lowerCAmelCase  :							List[Any]				     =					" ".join(__a).replace("@@ ",    "").strip()
														return out_string







							def      snake_case__   (     self,    __a,    __a = None):

														'''simple docstring'''

														if not os.path.isdir(__a):
																					logger.error(f"Vocabulary path ({save_directory}) should be a directory")
																					return
														_lowerCAmelCase  :							Dict				     =					os.path.join(
														    __a,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
														_lowerCAmelCase  :							Any				     =					os.path.join(
														    __a,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"])

														if os.path.abspath(self.vocab_file) != os.path.abspath(__a):
																					copyfile(self.vocab_file,    __a)

														if os.path.abspath(self.merges_file) != os.path.abspath(__a):
																					copyfile(self.merges_file,    __a)

														return out_vocab_file, out_merge_file







							def      snake_case__   (     self,    __a):

														'''simple docstring'''

														if isinstance(__a,    __a):
																					try:
																												with open(__a,    "r",    encoding="utf-8") as fd:
																																			self.add_from_file(__a)
																					except FileNotFoundError as fnfe:
																												raise fnfe
																					except UnicodeError:
																												raise Exception(f"Incorrect encoding detected in {f}, please rebuild the dataset")
																					return

														_lowerCAmelCase  :							Any				     =					f.readlines()
														for lineTmp in lines:
																					_lowerCAmelCase  :							Any				     =					lineTmp.strip()
																					_lowerCAmelCase  :							str				     =					line.rfind(" ")
																					if idx == -1:
																												raise ValueError("Incorrect dictionary format, expected '<token> <cnt>'")
																					_lowerCAmelCase  :							List[Any]				     =					line[:idx]
																					_lowerCAmelCase  :							List[str]				     =					len(self.encoder)





from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


_snake_case	=						logging.get_logger(__name__)

_snake_case	=						{
    "microsoft/focalnet-tiny": "https://huggingface.co/microsoft/focalnet-tiny/resolve/main/config.json",
}



class 						UpperCAmelCase_					( a     ,      a):
							lowerCamelCase__         =	'focalnet'







							def __init__(     self,    __a=224,    __a=4,    __a=3,    __a=96,    __a=False,    __a=[192, 384, 768, 768],    __a=[2, 2, 6, 2],    __a=[2, 2, 2, 2],    __a=[3, 3, 3, 3],    __a="gelu",    __a=4.0,    __a=0.0,    __a=0.1,    __a=False,    __a=1E-4,    __a=False,    __a=False,    __a=False,    __a=0.02,    __a=1E-5,    __a=32,    __a=None,    __a=None,    **__a,    ):

														'''simple docstring'''

														super().__init__(**__a)

														_lowerCAmelCase  :							str				     =					image_size
														_lowerCAmelCase  :							List[str]				     =					patch_size
														_lowerCAmelCase  :							List[Any]				     =					num_channels
														_lowerCAmelCase  :							Tuple				     =					embed_dim
														_lowerCAmelCase  :							List[Any]				     =					use_conv_embed
														_lowerCAmelCase  :							Any				     =					hidden_sizes
														_lowerCAmelCase  :							Tuple				     =					depths
														_lowerCAmelCase  :							Dict				     =					focal_levels
														_lowerCAmelCase  :							Optional[Any]				     =					focal_windows
														_lowerCAmelCase  :							str				     =					hidden_act
														_lowerCAmelCase  :							Union[str, Any]				     =					mlp_ratio
														_lowerCAmelCase  :							Any				     =					hidden_dropout_prob
														_lowerCAmelCase  :							Dict				     =					drop_path_rate
														_lowerCAmelCase  :							str				     =					use_layerscale
														_lowerCAmelCase  :							str				     =					layerscale_value
														_lowerCAmelCase  :							Union[str, Any]				     =					use_post_layernorm
														_lowerCAmelCase  :							Optional[int]				     =					use_post_layernorm_in_modulation
														_lowerCAmelCase  :							str				     =					normalize_modulator
														_lowerCAmelCase  :							Any				     =					initializer_range
														_lowerCAmelCase  :							Union[str, Any]				     =					layer_norm_eps
														_lowerCAmelCase  :							Any				     =					encoder_stride
														_lowerCAmelCase  :							List[str]				     =					["stem"] + [f"stage{idx}" for idx in range(1,    len(self.depths) + 1)]
														_lowerCAmelCase     ,					_lowerCAmelCase  :							List[str]				     =					get_aligned_output_features_output_indices(
														    out_features=__a,    out_indices=__a,    stage_names=self.stage_names)
