




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase : Optional[int] 			=				logging.get_logger(__name__)


__UpperCAmelCase : Union[str, Any] 			=				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

__UpperCAmelCase : Dict 			=				{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

__UpperCAmelCase : Dict 			=				{
    """allenai/longformer-base-4096""": 4096,
    """allenai/longformer-large-4096""": 4096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4096,
}


@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      a	(							):






       """simple docstring"""





       UpperCamelCase    :	List[Any]   	=		(
           list(range(ord('''!'''       )							,			ord('''~'''       ) + 1       )       ) + list(range(ord('''¡'''       )							,			ord('''¬'''       ) + 1       )       ) + list(range(ord('''®'''       )							,			ord('''ÿ'''       ) + 1       )       )
       )
       UpperCamelCase    :	Optional[int]   	=		bs[:]
       UpperCamelCase    :	List[Any]   	=		0
       for b in range(2**8       ):
              if b not in bs:
                     bs.append(_lowerCamelCase       )
                     cs.append(2**8 + n       )
                     n += 1
       UpperCamelCase    :	int   	=		[chr(_lowerCamelCase       ) for n in cs]
       return dict(zip(_lowerCamelCase							,			_lowerCamelCase       )       )


def      a	(							SCREAMING_SNAKE_CASE_      :		Optional[int]       ):






       """simple docstring"""





       UpperCamelCase    :	Optional[int]   	=		set()
       UpperCamelCase    :	str   	=		word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)       )
              UpperCamelCase    :	List[Any]   	=		char
       return pairs



class    UpperCAmelCase_	(							_a):
       '''simple docstring'''
       __UpperCamelCase      :     List[Any]							=					VOCAB_FILES_NAMES
       __UpperCamelCase      :     Optional[Any]							=					PRETRAINED_VOCAB_FILES_MAP
       __UpperCamelCase      :     int							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __UpperCamelCase      :     str							=					["input_ids", "attention_mask"]






       def __init__(     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE="replace" ,					__SCREAMING_SNAKE_CASE="<s>" ,					__SCREAMING_SNAKE_CASE="</s>" ,					__SCREAMING_SNAKE_CASE="</s>" ,					__SCREAMING_SNAKE_CASE="<s>" ,					__SCREAMING_SNAKE_CASE="<unk>" ,					__SCREAMING_SNAKE_CASE="<pad>" ,					__SCREAMING_SNAKE_CASE="<mask>" ,					__SCREAMING_SNAKE_CASE=False ,					**__SCREAMING_SNAKE_CASE ,					):




              """simple docstring"""




              UpperCamelCase    :	str   	=		AddedToken(UpperCamelCase__ ,					lstrip=UpperCamelCase__ ,					rstrip=UpperCamelCase__						) if isinstance(UpperCamelCase__ ,					UpperCamelCase__						) else bos_token
              UpperCamelCase    :	List[str]   	=		AddedToken(UpperCamelCase__ ,					lstrip=UpperCamelCase__ ,					rstrip=UpperCamelCase__						) if isinstance(UpperCamelCase__ ,					UpperCamelCase__						) else eos_token
              UpperCamelCase    :	Dict   	=		AddedToken(UpperCamelCase__ ,					lstrip=UpperCamelCase__ ,					rstrip=UpperCamelCase__						) if isinstance(UpperCamelCase__ ,					UpperCamelCase__						) else sep_token
              UpperCamelCase    :	str   	=		AddedToken(UpperCamelCase__ ,					lstrip=UpperCamelCase__ ,					rstrip=UpperCamelCase__						) if isinstance(UpperCamelCase__ ,					UpperCamelCase__						) else cls_token
              UpperCamelCase    :	List[str]   	=		AddedToken(UpperCamelCase__ ,					lstrip=UpperCamelCase__ ,					rstrip=UpperCamelCase__						) if isinstance(UpperCamelCase__ ,					UpperCamelCase__						) else unk_token
              UpperCamelCase    :	Optional[Any]   	=		AddedToken(UpperCamelCase__ ,					lstrip=UpperCamelCase__ ,					rstrip=UpperCamelCase__						) if isinstance(UpperCamelCase__ ,					UpperCamelCase__						) else pad_token

              # Mask token behave like a normal word, i.e. include the space before it
              UpperCamelCase    :	Dict   	=		AddedToken(UpperCamelCase__ ,					lstrip=UpperCamelCase__ ,					rstrip=UpperCamelCase__						) if isinstance(UpperCamelCase__ ,					UpperCamelCase__						) else mask_token

              super().__init__(
                  errors=UpperCamelCase__ ,					bos_token=UpperCamelCase__ ,					eos_token=UpperCamelCase__ ,					unk_token=UpperCamelCase__ ,					sep_token=UpperCamelCase__ ,					cls_token=UpperCamelCase__ ,					pad_token=UpperCamelCase__ ,					mask_token=UpperCamelCase__ ,					add_prefix_space=UpperCamelCase__ ,					**UpperCamelCase__ ,					)

              with open(UpperCamelCase__ ,					encoding='''utf-8'''						) as vocab_handle:
                     UpperCamelCase    :	List[str]   	=		json.load(UpperCamelCase__						)
              UpperCamelCase    :	Optional[Any]   	=		{v: k for k, v in self.encoder.items()}
              UpperCamelCase    :	List[Any]   	=		errors  # how to handle errors in decoding
              UpperCamelCase    :	int   	=		bytes_to_unicode()
              UpperCamelCase    :	List[Any]   	=		{v: k for k, v in self.byte_encoder.items()}
              with open(UpperCamelCase__ ,					encoding='''utf-8'''						) as merges_handle:
                     UpperCamelCase    :	Tuple   	=		merges_handle.read().split('''\n'''						)[1:-1]
              UpperCamelCase    :	str   	=		[tuple(merge.split()						) for merge in bpe_merges]
              UpperCamelCase    :	Dict   	=		dict(zip(UpperCamelCase__ ,					range(len(UpperCamelCase__						)						)						)						)
              UpperCamelCase    :	Optional[int]   	=		{}
              UpperCamelCase    :	Optional[int]   	=		add_prefix_space

              # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
              UpperCamelCase    :	str   	=		re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''						)






       @property
       def  _lowercase   (     self						):




              """simple docstring"""




              return len(self.encoder						)






       def  _lowercase   (     self						):




              """simple docstring"""




              return dict(self.encoder ,					**self.added_tokens_encoder						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              if token in self.cache:
                     return self.cache[token]
              UpperCamelCase    :	str   	=		tuple(UpperCamelCase__						)
              UpperCamelCase    :	Any   	=		get_pairs(UpperCamelCase__						)

              if not pairs:
                     return token

              while True:
                     UpperCamelCase    :	Dict   	=		min(UpperCamelCase__ ,					key=lambda __SCREAMING_SNAKE_CASE						: self.bpe_ranks.get(UpperCamelCase__ ,					float('''inf'''						)						)						)
                     if bigram not in self.bpe_ranks:
                            break
                     UpperCamelCase	, UpperCamelCase    :	str   	=		bigram
                     UpperCamelCase    :	Tuple   	=		[]
                     UpperCamelCase    :	int   	=		0
                     while i < len(UpperCamelCase__						):
                            try:
                                   UpperCamelCase    :	Union[str, Any]   	=		word.index(UpperCamelCase__ ,					UpperCamelCase__						)
                            except ValueError:
                                   new_word.extend(word[i:]						)
                                   break
                            else:
                                   new_word.extend(word[i:j]						)
                                   UpperCamelCase    :	Optional[Any]   	=		j

                            if word[i] == first and i < len(UpperCamelCase__						) - 1 and word[i + 1] == second:
                                   new_word.append(first + second						)
                                   i += 2
                            else:
                                   new_word.append(word[i]						)
                                   i += 1
                     UpperCamelCase    :	int   	=		tuple(UpperCamelCase__						)
                     UpperCamelCase    :	Optional[Any]   	=		new_word
                     if len(UpperCamelCase__						) == 1:
                            break
                     else:
                            UpperCamelCase    :	Dict   	=		get_pairs(UpperCamelCase__						)
              UpperCamelCase    :	Union[str, Any]   	=		''' '''.join(UpperCamelCase__						)
              UpperCamelCase    :	Dict   	=		word
              return word






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	str   	=		[]
              for token in re.findall(self.pat ,					UpperCamelCase__						):
                     UpperCamelCase    :	int   	=		''''''.join(
                         self.byte_encoder[b] for b in token.encode('''utf-8'''						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                     bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase__						).split(''' '''						)						)
              return bpe_tokens






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return self.encoder.get(UpperCamelCase__ ,					self.encoder.get(self.unk_token						)						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              return self.decoder.get(UpperCamelCase__						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Optional[Any]   	=		''''''.join(UpperCamelCase__						)
              UpperCamelCase    :	Optional[Any]   	=		bytearray([self.byte_decoder[c] for c in text]						).decode('''utf-8''' ,					errors=self.errors						)
              return text






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              if not os.path.isdir(UpperCamelCase__						):
                     logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
                     return
              UpperCamelCase    :	List[Any]   	=		os.path.join(
                  UpperCamelCase__ ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
              UpperCamelCase    :	Union[str, Any]   	=		os.path.join(
                  UpperCamelCase__ ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

              with open(UpperCamelCase__ ,					'''w''' ,					encoding='''utf-8'''						) as f:
                     f.write(json.dumps(self.encoder ,					indent=2 ,					sort_keys=UpperCamelCase__ ,					ensure_ascii=UpperCamelCase__						) + '''\n'''						)

              UpperCamelCase    :	List[str]   	=		0
              with open(UpperCamelCase__ ,					'''w''' ,					encoding='''utf-8'''						) as writer:
                     writer.write('''#version: 0.2\n'''						)
                     for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,					key=lambda __SCREAMING_SNAKE_CASE						: kv[1]						):
                            if index != token_index:
                                   logger.warning(
                                       f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                                       ''' Please check that the tokenizer is not corrupted!'''						)
                                   UpperCamelCase    :	str   	=		token_index
                            writer.write(''' '''.join(UpperCamelCase__						) + '''\n'''						)
                            index += 1

              return vocab_file, merge_file






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              if token_ids_a is None:
                     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
              UpperCamelCase    :	Tuple   	=		[self.cls_token_id]
              UpperCamelCase    :	Dict   	=		[self.sep_token_id]
              return cls + token_ids_a + sep + sep + token_ids_a + sep






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None ,					__SCREAMING_SNAKE_CASE = False						):




              """simple docstring"""




              if already_has_special_tokens:
                     return super().get_special_tokens_mask(
                         token_ids_a=UpperCamelCase__ ,					token_ids_a=UpperCamelCase__ ,					already_has_special_tokens=UpperCamelCase__						)

              if token_ids_a is None:
                     return [1] + ([0] * len(UpperCamelCase__						)) + [1]
              return [1] + ([0] * len(UpperCamelCase__						)) + [1, 1] + ([0] * len(UpperCamelCase__						)) + [1]






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              UpperCamelCase    :	str   	=		[self.sep_token_id]
              UpperCamelCase    :	int   	=		[self.cls_token_id]

              if token_ids_a is None:
                     return len(cls + token_ids_a + sep						) * [0]
              return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=False ,					**__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Tuple   	=		kwargs.pop('''add_prefix_space''' ,					self.add_prefix_space						)
              if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase__						) > 0 and not text[0].isspace()):
                     UpperCamelCase    :	Tuple   	=		''' ''' + text
              return (text, kwargs)





import copy
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


__UpperCAmelCase : Optional[Any] 			=				logging.get_logger(__name__)

__UpperCAmelCase : List[str] 			=				{
    "microsoft/conditional-detr-resnet-50": (
        "https://huggingface.co/microsoft/conditional-detr-resnet-50/resolve/main/config.json"
    ),
}



class    UpperCAmelCase_	(							_a):
       '''simple docstring'''
       __UpperCamelCase      :     Optional[int]							=					"conditional_detr"
       __UpperCamelCase      :     Optional[Any]							=					["past_key_values"]
       __UpperCamelCase      :     Union[str, Any]							=					{
           "hidden_size": "d_model",
           "num_attention_heads": "encoder_attention_heads",
       }






       def __init__(     self ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=3 ,					__SCREAMING_SNAKE_CASE=300 ,					__SCREAMING_SNAKE_CASE=6 ,					__SCREAMING_SNAKE_CASE=2_048 ,					__SCREAMING_SNAKE_CASE=8 ,					__SCREAMING_SNAKE_CASE=6 ,					__SCREAMING_SNAKE_CASE=2_048 ,					__SCREAMING_SNAKE_CASE=8 ,					__SCREAMING_SNAKE_CASE=0.0 ,					__SCREAMING_SNAKE_CASE=0.0 ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE="relu" ,					__SCREAMING_SNAKE_CASE=256 ,					__SCREAMING_SNAKE_CASE=0.1 ,					__SCREAMING_SNAKE_CASE=0.0 ,					__SCREAMING_SNAKE_CASE=0.0 ,					__SCREAMING_SNAKE_CASE=0.02 ,					__SCREAMING_SNAKE_CASE=1.0 ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE="sine" ,					__SCREAMING_SNAKE_CASE="resnet50" ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE=False ,					__SCREAMING_SNAKE_CASE=2 ,					__SCREAMING_SNAKE_CASE=5 ,					__SCREAMING_SNAKE_CASE=2 ,					__SCREAMING_SNAKE_CASE=1 ,					__SCREAMING_SNAKE_CASE=1 ,					__SCREAMING_SNAKE_CASE=2 ,					__SCREAMING_SNAKE_CASE=5 ,					__SCREAMING_SNAKE_CASE=2 ,					__SCREAMING_SNAKE_CASE=0.25 ,					**__SCREAMING_SNAKE_CASE ,					):




              """simple docstring"""




              if backbone_config is not None and use_timm_backbone:
                     raise ValueError('''You can\'t specify both `backbone_config` and `use_timm_backbone`.'''						)

              if not use_timm_backbone:
                     if backbone_config is None:
                            logger.info('''`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone.'''						)
                            UpperCamelCase    :	str   	=		CONFIG_MAPPING['''resnet'''](out_features=['''stage4''']						)
                     elif isinstance(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):
                            UpperCamelCase    :	Tuple   	=		backbone_config.get('''model_type'''						)
                            UpperCamelCase    :	Optional[Any]   	=		CONFIG_MAPPING[backbone_model_type]
                            UpperCamelCase    :	Any   	=		config_class.from_dict(__SCREAMING_SNAKE_CASE						)

              UpperCamelCase    :	List[str]   	=		use_timm_backbone
              UpperCamelCase    :	int   	=		backbone_config
              UpperCamelCase    :	Any   	=		num_channels
              UpperCamelCase    :	Optional[Any]   	=		num_queries
              UpperCamelCase    :	Tuple   	=		d_model
              UpperCamelCase    :	Optional[Any]   	=		encoder_ffn_dim
              UpperCamelCase    :	Optional[int]   	=		encoder_layers
              UpperCamelCase    :	Union[str, Any]   	=		encoder_attention_heads
              UpperCamelCase    :	Optional[Any]   	=		decoder_ffn_dim
              UpperCamelCase    :	Optional[int]   	=		decoder_layers
              UpperCamelCase    :	Optional[Any]   	=		decoder_attention_heads
              UpperCamelCase    :	Any   	=		dropout
              UpperCamelCase    :	List[Any]   	=		attention_dropout
              UpperCamelCase    :	List[Any]   	=		activation_dropout
              UpperCamelCase    :	List[str]   	=		activation_function
              UpperCamelCase    :	Optional[int]   	=		init_std
              UpperCamelCase    :	Optional[Any]   	=		init_xavier_std
              UpperCamelCase    :	Union[str, Any]   	=		encoder_layerdrop
              UpperCamelCase    :	Optional[Any]   	=		decoder_layerdrop
              UpperCamelCase    :	Tuple   	=		encoder_layers
              UpperCamelCase    :	Optional[Any]   	=		auxiliary_loss
              UpperCamelCase    :	Union[str, Any]   	=		position_embedding_type
              UpperCamelCase    :	Optional[int]   	=		backbone
              UpperCamelCase    :	Dict   	=		use_pretrained_backbone
              UpperCamelCase    :	Tuple   	=		dilation
              # Hungarian matcher
              UpperCamelCase    :	Union[str, Any]   	=		class_cost
              UpperCamelCase    :	List[Any]   	=		bbox_cost
              UpperCamelCase    :	Optional[Any]   	=		giou_cost
              # Loss coefficients
              UpperCamelCase    :	Optional[Any]   	=		mask_loss_coefficient
              UpperCamelCase    :	Optional[int]   	=		dice_loss_coefficient
              UpperCamelCase    :	Optional[Any]   	=		cls_loss_coefficient
              UpperCamelCase    :	Optional[int]   	=		bbox_loss_coefficient
              UpperCamelCase    :	Optional[int]   	=		giou_loss_coefficient
              UpperCamelCase    :	Optional[int]   	=		focal_alpha
              super().__init__(is_encoder_decoder=__SCREAMING_SNAKE_CASE ,					**__SCREAMING_SNAKE_CASE						)






       @property
       def  _lowercase   (     self						):




              """simple docstring"""




              return self.encoder_attention_heads






       @property
       def  _lowercase   (     self						):




              """simple docstring"""




              return self.d_model






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	Union[str, Any]   	=		copy.deepcopy(self.__dict__						)
              if self.backbone_config is not None:
                     UpperCamelCase    :	List[Any]   	=		self.backbone_config.to_dict()
              UpperCamelCase    :	List[Any]   	=		self.__class__.model_type
              return output




class    UpperCAmelCase_	(							_a):
       '''simple docstring'''
       __UpperCamelCase      :     Dict							=					version.parse("1.11")






       @property
       def  _lowercase   (     self						):




              """simple docstring"""




              return OrderedDict(
                  [
                      ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
                      ('''pixel_mask''', {0: '''batch'''}),
                  ]						)






       @property
       def  _lowercase   (     self						):




              """simple docstring"""




              return 1e-5






       @property
       def  _lowercase   (     self						):




              """simple docstring"""




              return 12

