



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_snake_case	=						logging.get_logger(__name__)

_snake_case	=						{'vocab_file': 'sentencepiece.bpe.model'}

_snake_case	=						{
    'vocab_file': {
        'camembert-base': 'https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model',
    }
}

_snake_case	=						{
    'camembert-base': 512,
}

_snake_case	=						'▁'



class 						UpperCAmelCase_					( a):
       lowerCamelCase__         =	VOCAB_FILES_NAMES
       lowerCamelCase__         =	PRETRAINED_VOCAB_FILES_MAP
       lowerCamelCase__         =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCamelCase__         =	['input_ids', 'attention_mask']







       def __init__(     self,    __a,    __a="<s>",    __a="</s>",    __a="</s>",    __a="<s>",    __a="<unk>",    __a="<pad>",    __a="<mask>",    __a=["<s>NOTUSED", "</s>NOTUSED"],    __a = None,    **__a,    ):

              '''simple docstring'''

              _lowerCAmelCase  :							List[str]				     =					AddedToken(_lowerCAmelCase,    lstrip=_lowerCAmelCase,    rstrip=_lowerCAmelCase) if isinstance(_lowerCAmelCase,    _lowerCAmelCase) else mask_token

              _lowerCAmelCase  :							Any				     =					{} if sp_model_kwargs is None else sp_model_kwargs

              super().__init__(
                  bos_token=_lowerCAmelCase,    eos_token=_lowerCAmelCase,    unk_token=_lowerCAmelCase,    sep_token=_lowerCAmelCase,    cls_token=_lowerCAmelCase,    pad_token=_lowerCAmelCase,    mask_token=_lowerCAmelCase,    additional_special_tokens=_lowerCAmelCase,    sp_model_kwargs=self.sp_model_kwargs,    **_lowerCAmelCase,    )
              _lowerCAmelCase  :							List[str]				     =					spm.SentencePieceProcessor(**self.sp_model_kwargs)
              self.sp_model.Load(str(_lowerCAmelCase))
              _lowerCAmelCase  :							Optional[Any]				     =					vocab_file
              # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
              # sentencepiece vocabulary (this is the case for <s> and </s>
              _lowerCAmelCase  :							int				     =					{"<s>NOTUSED": 0, "<pad>": 1, "</s>NOTUSED": 2, "<unk>": 3}
              _lowerCAmelCase  :							Tuple				     =					len(self.fairseq_tokens_to_ids)
              _lowerCAmelCase  :							Optional[Any]				     =					len(self.sp_model) + len(self.fairseq_tokens_to_ids)
              _lowerCAmelCase  :							Dict				     =					{v: k for k, v in self.fairseq_tokens_to_ids.items()}







       def      snake_case__   (     self,    __a,    __a = None):

              '''simple docstring'''

              if token_ids_a is None:
                     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
              _lowerCAmelCase  :							Any				     =					[self.cls_token_id]
              _lowerCAmelCase  :							List[str]				     =					[self.sep_token_id]
              return cls + token_ids_a + sep + sep + token_ids_a + sep







       def      snake_case__   (     self,    __a,    __a = None,    __a = False):

              '''simple docstring'''

              if already_has_special_tokens:
                     return super().get_special_tokens_mask(
                         token_ids_a=_lowerCAmelCase,    token_ids_a=_lowerCAmelCase,    already_has_special_tokens=_lowerCAmelCase)

              if token_ids_a is None:
                     return [1] + ([0] * len(_lowerCAmelCase)) + [1]
              return [1] + ([0] * len(_lowerCAmelCase)) + [1, 1] + ([0] * len(_lowerCAmelCase)) + [1]







       def      snake_case__   (     self,    __a,    __a = None):

              '''simple docstring'''

              _lowerCAmelCase  :							int				     =					[self.sep_token_id]
              _lowerCAmelCase  :							Tuple				     =					[self.cls_token_id]

              if token_ids_a is None:
                     return len(cls + token_ids_a + sep) * [0]
              return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]







       @property
       def      snake_case__   (     self):

              '''simple docstring'''

              return len(self.fairseq_tokens_to_ids) + len(self.sp_model)







       def      snake_case__   (     self):

              '''simple docstring'''

              _lowerCAmelCase  :							List[str]				     =					{self.convert_ids_to_tokens(_lowerCAmelCase): i for i in range(self.vocab_size)}
              vocab.update(self.added_tokens_encoder)
              return vocab







       def      snake_case__   (     self,    __a):

              '''simple docstring'''

              return self.sp_model.encode(_lowerCAmelCase,    out_type=_lowerCAmelCase)







       def      snake_case__   (     self,    __a):

              '''simple docstring'''

              if token in self.fairseq_tokens_to_ids:
                     return self.fairseq_tokens_to_ids[token]
              elif self.sp_model.PieceToId(_lowerCAmelCase) == 0:
                     # Convert sentence piece unk token to fairseq unk token index
                     return self.unk_token_id
              return self.fairseq_offset + self.sp_model.PieceToId(_lowerCAmelCase)







       def      snake_case__   (     self,    __a):

              '''simple docstring'''

              if index in self.fairseq_ids_to_tokens:
                     return self.fairseq_ids_to_tokens[index]
              return self.sp_model.IdToPiece(index - self.fairseq_offset)







       def      snake_case__   (     self,    __a):

              '''simple docstring'''

              _lowerCAmelCase  :							Tuple				     =					[]
              _lowerCAmelCase  :							Optional[int]				     =					""
              _lowerCAmelCase  :							Optional[Any]				     =					False
              for token in tokens:
                     # make sure that special tokens are not decoded using sentencepiece model
                     if token in self.all_special_tokens:
                            if not prev_is_special:
                                   out_string += " "
                            out_string += self.sp_model.decode(_lowerCAmelCase) + token
                            _lowerCAmelCase  :							Union[str, Any]				     =					True
                            _lowerCAmelCase  :							Tuple				     =					[]
                     else:
                            current_sub_tokens.append(_lowerCAmelCase)
                            _lowerCAmelCase  :							List[str]				     =					False
              out_string += self.sp_model.decode(_lowerCAmelCase)
              return out_string.strip()







       def __getstate__(     self):

              '''simple docstring'''

              _lowerCAmelCase  :							Dict				     =					self.__dict__.copy()
              _lowerCAmelCase  :							List[str]				     =					None
              return state







       def __setstate__(     self,    __a):

              '''simple docstring'''

              _lowerCAmelCase  :							Optional[Any]				     =					d

              # for backward compatibility
              if not hasattr(self,    "sp_model_kwargs"):
                     _lowerCAmelCase  :							List[Any]				     =					{}

              _lowerCAmelCase  :							int				     =					spm.SentencePieceProcessor(**self.sp_model_kwargs)
              self.sp_model.Load(self.vocab_file)







       def      snake_case__   (     self,    __a,    __a = None):

              '''simple docstring'''

              if not os.path.isdir(_lowerCAmelCase):
                     logger.error(f"Vocabulary path ({save_directory}) should be a directory")
                     return
              _lowerCAmelCase  :							Optional[int]				     =					os.path.join(
                  _lowerCAmelCase,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

              if os.path.abspath(self.vocab_file) != os.path.abspath(_lowerCAmelCase) and os.path.isfile(self.vocab_file):
                     copyfile(self.vocab_file,    _lowerCAmelCase)
              elif not os.path.isfile(self.vocab_file):
                     with open(_lowerCAmelCase,    "wb") as fi:
                            _lowerCAmelCase  :							Optional[Any]				     =					self.sp_model.serialized_model_proto()
                            fi.write(_lowerCAmelCase)

              return (out_vocab_file,)





from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


_snake_case	=						logging.get_logger(__name__)

_snake_case	=						{
    "microsoft/focalnet-tiny": "https://huggingface.co/microsoft/focalnet-tiny/resolve/main/config.json",
}



class 						UpperCAmelCase_					( a     ,      a):
							lowerCamelCase__         =	'focalnet'







							def __init__(     self,    __a=224,    __a=4,    __a=3,    __a=96,    __a=False,    __a=[192, 384, 768, 768],    __a=[2, 2, 6, 2],    __a=[2, 2, 2, 2],    __a=[3, 3, 3, 3],    __a="gelu",    __a=4.0,    __a=0.0,    __a=0.1,    __a=False,    __a=1E-4,    __a=False,    __a=False,    __a=False,    __a=0.02,    __a=1E-5,    __a=32,    __a=None,    __a=None,    **__a,    ):

														'''simple docstring'''

														super().__init__(**__a)

														_lowerCAmelCase  :							str				     =					image_size
														_lowerCAmelCase  :							List[str]				     =					patch_size
														_lowerCAmelCase  :							List[Any]				     =					num_channels
														_lowerCAmelCase  :							Tuple				     =					embed_dim
														_lowerCAmelCase  :							List[Any]				     =					use_conv_embed
														_lowerCAmelCase  :							Any				     =					hidden_sizes
														_lowerCAmelCase  :							Tuple				     =					depths
														_lowerCAmelCase  :							Dict				     =					focal_levels
														_lowerCAmelCase  :							Optional[Any]				     =					focal_windows
														_lowerCAmelCase  :							str				     =					hidden_act
														_lowerCAmelCase  :							Union[str, Any]				     =					mlp_ratio
														_lowerCAmelCase  :							Any				     =					hidden_dropout_prob
														_lowerCAmelCase  :							Dict				     =					drop_path_rate
														_lowerCAmelCase  :							str				     =					use_layerscale
														_lowerCAmelCase  :							str				     =					layerscale_value
														_lowerCAmelCase  :							Union[str, Any]				     =					use_post_layernorm
														_lowerCAmelCase  :							Optional[int]				     =					use_post_layernorm_in_modulation
														_lowerCAmelCase  :							str				     =					normalize_modulator
														_lowerCAmelCase  :							Any				     =					initializer_range
														_lowerCAmelCase  :							Union[str, Any]				     =					layer_norm_eps
														_lowerCAmelCase  :							Any				     =					encoder_stride
														_lowerCAmelCase  :							List[str]				     =					["stem"] + [f"stage{idx}" for idx in range(1,    len(self.depths) + 1)]
														_lowerCAmelCase     ,					_lowerCAmelCase  :							List[str]				     =					get_aligned_output_features_output_indices(
														    out_features=__a,    out_indices=__a,    stage_names=self.stage_names)
