



import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a__      :				Optional[Any]   	=			logging.get_logger(__name__)


a__      :				str   	=			{"vocab_file": "vocab.json", "merges_file": "merges.txt"}

a__      :				Tuple   	=			{
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
}

a__      :				Dict   	=			{
    "allenai/longformer-base-4096": 40_96,
    "allenai/longformer-large-4096": 40_96,
    "allenai/longformer-large-4096-finetuned-triviaqa": 40_96,
    "allenai/longformer-base-4096-extra.pos.embd.only": 40_96,
    "allenai/longformer-large-4096-extra.pos.embd.only": 40_96,
}

@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def   _lowerCAmelCase   (						):
      lowercase__     =  (
          list(range(ord('!'	)						,  ord('~'	) + 1	)	) + list(range(ord('¡'	)						,  ord('¬'	) + 1	)	) + list(range(ord('®'	)						,  ord('ÿ'	) + 1	)	)
      )
      lowercase__     =  bs[:]
      lowercase__     =  0
      for b in range(2**8	):
            if b not in bs:
                  bs.append(a_	)
                  cs.append(2**8 + n	)
                  n += 1
      lowercase__     =  [chr(a_	) for n in cs]
      return dict(zip(a_						,  a_	)	)

def   _lowerCAmelCase   (						A__	):
      lowercase__     =  set()
      lowercase__     =  word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)	)
            lowercase__     =  char
      return pairs



class       UpperCAmelCase__(							__SCREAMING_SNAKE_CASE	):

   '''simple docstring'''




   A    :  Any       				=    VOCAB_FILES_NAMES
   A    :  str       				=    PRETRAINED_VOCAB_FILES_MAP
   A    :  Optional[int]       				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A    :  str       				=    ["input_ids", "attention_mask"]


   def __init__(						self  :			Optional[int]			,				lowerCAmelCase  :			Optional[int]			,				lowerCAmelCase  :			Any			,				lowerCAmelCase  :			Dict="replace"			,				lowerCAmelCase  :			List[Any]="<s>"			,				lowerCAmelCase  :			Optional[int]="</s>"			,				lowerCAmelCase  :			int="</s>"			,				lowerCAmelCase  :			Any="<s>"			,				lowerCAmelCase  :			List[str]="<unk>"			,				lowerCAmelCase  :			Optional[Any]="<pad>"			,				lowerCAmelCase  :			Optional[Any]="<mask>"			,				lowerCAmelCase  :			Union[str, Any]=False			,				**lowerCAmelCase  :			int			,				)	->				Any:




         """simple docstring"""

         lowercase__     =  AddedToken(__snake_case			,				lstrip=__snake_case			,				rstrip=__snake_case    ) if isinstance(__snake_case			,				__snake_case    ) else bos_token
         lowercase__     =  AddedToken(__snake_case			,				lstrip=__snake_case			,				rstrip=__snake_case    ) if isinstance(__snake_case			,				__snake_case    ) else eos_token
         lowercase__     =  AddedToken(__snake_case			,				lstrip=__snake_case			,				rstrip=__snake_case    ) if isinstance(__snake_case			,				__snake_case    ) else sep_token
         lowercase__     =  AddedToken(__snake_case			,				lstrip=__snake_case			,				rstrip=__snake_case    ) if isinstance(__snake_case			,				__snake_case    ) else cls_token
         lowercase__     =  AddedToken(__snake_case			,				lstrip=__snake_case			,				rstrip=__snake_case    ) if isinstance(__snake_case			,				__snake_case    ) else unk_token
         lowercase__     =  AddedToken(__snake_case			,				lstrip=__snake_case			,				rstrip=__snake_case    ) if isinstance(__snake_case			,				__snake_case    ) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         lowercase__     =  AddedToken(__snake_case			,				lstrip=__snake_case			,				rstrip=__snake_case    ) if isinstance(__snake_case			,				__snake_case    ) else mask_token

         super().__init__(
             errors=__snake_case			,				bos_token=__snake_case			,				eos_token=__snake_case			,				unk_token=__snake_case			,				sep_token=__snake_case			,				cls_token=__snake_case			,				pad_token=__snake_case			,				mask_token=__snake_case			,				add_prefix_space=__snake_case			,				**__snake_case			,				)

         with open(__snake_case			,				encoding='utf-8'    ) as vocab_handle:
               lowercase__     =  json.load(__snake_case    )
         lowercase__     =  {v: k for k, v in self.encoder.items()}
         lowercase__     =  errors  # how to handle errors in decoding
         lowercase__     =  bytes_to_unicode()
         lowercase__     =  {v: k for k, v in self.byte_encoder.items()}
         with open(__snake_case			,				encoding='utf-8'    ) as merges_handle:
               lowercase__     =  merges_handle.read().split('\n'    )[1:-1]
         lowercase__     =  [tuple(merge.split()    ) for merge in bpe_merges]
         lowercase__     =  dict(zip(__snake_case			,				range(len(__snake_case    )    )    )    )
         lowercase__     =  {}
         lowercase__     =  add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         lowercase__     =  re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'    )


   @property
   def 			UpperCAmelCase		(						self  :			Any    )	->				int:




         """simple docstring"""

         return len(self.encoder    )


   def 			UpperCAmelCase		(						self  :			Any    )	->				Tuple:




         """simple docstring"""

         return dict(self.encoder			,				**self.added_tokens_encoder    )


   def 			UpperCAmelCase		(						self  :			List[str]			,				lowerCAmelCase  :			Tuple    )	->				Optional[Any]:




         """simple docstring"""

         if token in self.cache:
               return self.cache[token]
         lowercase__     =  tuple(__snake_case    )
         lowercase__     =  get_pairs(__snake_case    )

         if not pairs:
               return token

         while True:
               lowercase__     =  min(__snake_case			,				key=lambda lowerCAmelCase    : self.bpe_ranks.get(__snake_case			,				float('inf'    )    )    )
               if bigram not in self.bpe_ranks:
                     break
               lowercase__     =  bigram
               lowercase__     =  []
               lowercase__     =  0
               while i < len(__snake_case    ):
                     try:
                           lowercase__     =  word.index(__snake_case			,				__snake_case    )
                     except ValueError:
                           new_word.extend(word[i:]    )
                           break
                     else:
                           new_word.extend(word[i:j]    )
                           lowercase__     =  j

                     if word[i] == first and i < len(__snake_case    ) - 1 and word[i + 1] == second:
                           new_word.append(first + second    )
                           i += 2
                     else:
                           new_word.append(word[i]    )
                           i += 1
               lowercase__     =  tuple(__snake_case    )
               lowercase__     =  new_word
               if len(__snake_case    ) == 1:
                     break
               else:
                     lowercase__     =  get_pairs(__snake_case    )
         lowercase__     =  ''' '''.join(__snake_case    )
         lowercase__     =  word
         return word


   def 			UpperCAmelCase		(						self  :			List[str]			,				lowerCAmelCase  :			str    )	->				List[Any]:




         """simple docstring"""

         lowercase__     =  []
         for token in re.findall(self.pat			,				__snake_case    ):
               lowercase__     =  ''''''.join(
                   self.byte_encoder[b] for b in token.encode('utf-8'    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__snake_case    ).split(' '    )    )
         return bpe_tokens


   def 			UpperCAmelCase		(						self  :			Optional[int]			,				lowerCAmelCase  :			Tuple    )	->				Tuple:




         """simple docstring"""

         return self.encoder.get(__snake_case			,				self.encoder.get(self.unk_token    )    )


   def 			UpperCAmelCase		(						self  :			str			,				lowerCAmelCase  :			List[Any]    )	->				Optional[int]:




         """simple docstring"""

         return self.decoder.get(__snake_case    )


   def 			UpperCAmelCase		(						self  :			Any			,				lowerCAmelCase  :			Any    )	->				str:




         """simple docstring"""

         lowercase__     =  ''''''.join(__snake_case    )
         lowercase__     =  bytearray([self.byte_decoder[c] for c in text]    ).decode('utf-8'			,				errors=self.errors    )
         return text


   def 			UpperCAmelCase		(						self  :			List[str]			,				lowerCAmelCase  :			str			,				lowerCAmelCase  :			Optional[str] = None    )	->				Dict:




         """simple docstring"""

         if not os.path.isdir(__snake_case    ):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
               return
         lowercase__     =  os.path.join(
             __snake_case			,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
         lowercase__     =  os.path.join(
             __snake_case			,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']    )

         with open(__snake_case			,				'w'			,				encoding='utf-8'    ) as f:
               f.write(json.dumps(self.encoder			,				indent=2			,				sort_keys=__snake_case			,				ensure_ascii=__snake_case    ) + '\n'    )

         lowercase__     =  0
         with open(__snake_case			,				'w'			,				encoding='utf-8'    ) as writer:
               writer.write('#version: 0.2\n'    )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,				key=lambda lowerCAmelCase    : kv[1]    ):
                     if index != token_index:
                           logger.warning(
                               f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                               ' Please check that the tokenizer is not corrupted!'    )
                           lowercase__     =  token_index
                     writer.write(' '.join(__snake_case    ) + '\n'    )
                     index += 1

         return vocab_file, merge_file


   def 			UpperCAmelCase		(						self  :			Dict			,				lowerCAmelCase  :			List[int]			,				lowerCAmelCase  :			Optional[List[int]] = None    )	->				Optional[Any]:




         """simple docstring"""

         if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         lowercase__     =  [self.cls_token_id]
         lowercase__     =  [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep


   def 			UpperCAmelCase		(						self  :			Dict			,				lowerCAmelCase  :			List[int]			,				lowerCAmelCase  :			Optional[List[int]] = None			,				lowerCAmelCase  :			bool = False    )	->				int:




         """simple docstring"""

         if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__snake_case			,				token_ids_a=__snake_case			,				already_has_special_tokens=__snake_case    )

         if token_ids_a is None:
               return [1] + ([0] * len(__snake_case    )) + [1]
         return [1] + ([0] * len(__snake_case    )) + [1, 1] + ([0] * len(__snake_case    )) + [1]


   def 			UpperCAmelCase		(						self  :			Optional[int]			,				lowerCAmelCase  :			List[int]			,				lowerCAmelCase  :			Optional[List[int]] = None    )	->				Union[str, Any]:




         """simple docstring"""

         lowercase__     =  [self.sep_token_id]
         lowercase__     =  [self.cls_token_id]

         if token_ids_a is None:
               return len(cls + token_ids_a + sep    ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]



   def 			UpperCAmelCase		(						self  :			Tuple			,				lowerCAmelCase  :			Optional[Any]			,				lowerCAmelCase  :			Tuple=False			,				**lowerCAmelCase  :			Any    )	->				int:




         """simple docstring"""

         lowercase__     =  kwargs.pop('add_prefix_space'			,				self.add_prefix_space    )
         if (is_split_into_words or add_prefix_space) and (len(__snake_case    ) > 0 and not text[0].isspace()):
               lowercase__     =  ''' ''' + text
         return (text, kwargs)





from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


a__      :				List[str]   	=			logging.get_logger(__name__)

a__      :				List[Any]   	=			{
    "microsoft/focalnet-tiny": "https://huggingface.co/microsoft/focalnet-tiny/resolve/main/config.json",
}



class       UpperCAmelCase__(							lowerCamelCase     ,							lowerCamelCase	):

   '''simple docstring'''




   A    :  List[str]       				=    "focalnet"


   def __init__(						self  :			Dict			,				lowerCAmelCase  :			Union[str, Any]=2_24			,				lowerCAmelCase  :			List[str]=4			,				lowerCAmelCase  :			int=3			,				lowerCAmelCase  :			Union[str, Any]=96			,				lowerCAmelCase  :			List[Any]=False			,				lowerCAmelCase  :			int=[1_92, 3_84, 7_68, 7_68]			,				lowerCAmelCase  :			str=[2, 2, 6, 2]			,				lowerCAmelCase  :			Tuple=[2, 2, 2, 2]			,				lowerCAmelCase  :			Optional[Any]=[3, 3, 3, 3]			,				lowerCAmelCase  :			int="gelu"			,				lowerCAmelCase  :			Any=4.0			,				lowerCAmelCase  :			List[str]=0.0			,				lowerCAmelCase  :			Union[str, Any]=0.1			,				lowerCAmelCase  :			Union[str, Any]=False			,				lowerCAmelCase  :			Tuple=1E-4			,				lowerCAmelCase  :			List[Any]=False			,				lowerCAmelCase  :			Optional[int]=False			,				lowerCAmelCase  :			List[str]=False			,				lowerCAmelCase  :			str=0.02			,				lowerCAmelCase  :			Optional[int]=1E-5			,				lowerCAmelCase  :			List[Any]=32			,				lowerCAmelCase  :			List[Any]=None			,				lowerCAmelCase  :			Union[str, Any]=None			,				**lowerCAmelCase  :			str			,				)	->				List[str]:




         """simple docstring"""

         super().__init__(**lowerCAmelCase)

         lowercase__     =  image_size
         lowercase__     =  patch_size
         lowercase__     =  num_channels
         lowercase__     =  embed_dim
         lowercase__     =  use_conv_embed
         lowercase__     =  hidden_sizes
         lowercase__     =  depths
         lowercase__     =  focal_levels
         lowercase__     =  focal_windows
         lowercase__     =  hidden_act
         lowercase__     =  mlp_ratio
         lowercase__     =  hidden_dropout_prob
         lowercase__     =  drop_path_rate
         lowercase__     =  use_layerscale
         lowercase__     =  layerscale_value
         lowercase__     =  use_post_layernorm
         lowercase__     =  use_post_layernorm_in_modulation
         lowercase__     =  normalize_modulator
         lowercase__     =  initializer_range
         lowercase__     =  layer_norm_eps
         lowercase__     =  encoder_stride
         lowercase__     =  ['stem'] + [f'''stage{idx}''' for idx in range(1			,				len(self.depths) + 1)]
         lowercase__,	lowercase__     =  get_aligned_output_features_output_indices(
             out_features=lowerCAmelCase			,				out_indices=lowerCAmelCase			,				stage_names=self.stage_names)
