
import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast





class  snake_case     (							unittest.TestCase   ):

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = mock.Mock()
									SCREAMING_SNAKE_CASE_						 = 5_00
									SCREAMING_SNAKE_CASE_						 = {}
									SCREAMING_SNAKE_CASE_						 = HTTPError
									SCREAMING_SNAKE_CASE_						 = {}

									# Download this model to make sure it's in the cache.
									SCREAMING_SNAKE_CASE_						 = BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch('''requests.Session.request'''       ,       return_value=_lowercase    ) as mock_head:
											SCREAMING_SNAKE_CASE_						 = BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )
											# This check we did call the fake head request
											mock_head.assert_called()

							@require_tokenizers
							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = mock.Mock()
									SCREAMING_SNAKE_CASE_						 = 5_00
									SCREAMING_SNAKE_CASE_						 = {}
									SCREAMING_SNAKE_CASE_						 = HTTPError
									SCREAMING_SNAKE_CASE_						 = {}

									# Download this model to make sure it's in the cache.
									SCREAMING_SNAKE_CASE_						 = GPTaTokenizerFast.from_pretrained('''gpt2'''    )

									# Under the mock environment we get a 500 error when trying to reach the tokenizer.
									with mock.patch('''requests.Session.request'''       ,       return_value=_lowercase    ) as mock_head:
											SCREAMING_SNAKE_CASE_						 = GPTaTokenizerFast.from_pretrained('''gpt2'''    )
											# This check we did call the fake head request
											mock_head.assert_called()

							def _lowercase					(self    ):

									"""simple docstring"""







									try:
											SCREAMING_SNAKE_CASE_						 = tempfile.mktemp()
											with open(_lowercase       ,       '''wb'''    ) as f:
													http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''       ,       _lowercase    )

											SCREAMING_SNAKE_CASE_						 = AlbertTokenizer.from_pretrained(_lowercase    )
									finally:
											os.remove(_lowercase    )

									# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
									# the current folder and have the right name.
									if os.path.isfile('''tokenizer.json'''    ):
											# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
											return
									try:
											with open('''tokenizer.json'''       ,       '''wb'''    ) as f:
													http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'''       ,       _lowercase    )
											SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''    )
											# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
											self.assertEqual(tokenizer.vocab_size       ,       10_00    )
											# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

									finally:
											os.remove('''tokenizer.json'''    )







							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''    )







@is_staging_test
class  snake_case     (							unittest.TestCase   ):
							UpperCAmelCase__						     =      ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']

							@classmethod
							def _lowercase					(cls    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = TOKEN
									HfFolder.save_token(_lowercase    )

							@classmethod
							def _lowercase					(cls    ):

									"""simple docstring"""







									try:
											delete_repo(token=cls._token       ,       repo_id='''test-tokenizer'''    )
									except HTTPError:
											pass

									try:
											delete_repo(token=cls._token       ,       repo_id='''valid_org/test-tokenizer-org'''    )
									except HTTPError:
											pass

									try:
											delete_repo(token=cls._token       ,       repo_id='''test-dynamic-tokenizer'''    )
									except HTTPError:
											pass

							def _lowercase					(self    ):

									"""simple docstring"""







									with tempfile.TemporaryDirectory() as tmp_dir:
											SCREAMING_SNAKE_CASE_						 = os.path.join(_lowercase       ,       '''vocab.txt'''    )
											with open(_lowercase       ,       '''w'''       ,       encoding='''utf-8'''    ) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]    )    )
											SCREAMING_SNAKE_CASE_						 = BertTokenizer(_lowercase    )

									tokenizer.push_to_hub('''test-tokenizer'''       ,       use_auth_token=self._token    )
									SCREAMING_SNAKE_CASE_						 = BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'    )
									self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab    )

									# Reset repo
									delete_repo(token=self._token       ,       repo_id='''test-tokenizer'''    )

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(_lowercase       ,       repo_id='''test-tokenizer'''       ,       push_to_hub=_lowercase       ,       use_auth_token=self._token    )

									SCREAMING_SNAKE_CASE_						 = BertTokenizer.from_pretrained(f'{USER}/test-tokenizer'    )
									self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab    )

							def _lowercase					(self    ):

									"""simple docstring"""







									with tempfile.TemporaryDirectory() as tmp_dir:
											SCREAMING_SNAKE_CASE_						 = os.path.join(_lowercase       ,       '''vocab.txt'''    )
											with open(_lowercase       ,       '''w'''       ,       encoding='''utf-8'''    ) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]    )    )
											SCREAMING_SNAKE_CASE_						 = BertTokenizer(_lowercase    )

									tokenizer.push_to_hub('''valid_org/test-tokenizer-org'''       ,       use_auth_token=self._token    )
									SCREAMING_SNAKE_CASE_						 = BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''    )
									self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab    )

									# Reset repo
									delete_repo(token=self._token       ,       repo_id='''valid_org/test-tokenizer-org'''    )

									# Push to hub via save_pretrained
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(
											    _lowercase       ,       repo_id='''valid_org/test-tokenizer-org'''       ,       push_to_hub=_lowercase       ,       use_auth_token=self._token    )

									SCREAMING_SNAKE_CASE_						 = BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''    )
									self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab    )







							@require_tokenizers
							def _lowercase					(self    ):

									"""simple docstring"""







									CustomTokenizer.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
											SCREAMING_SNAKE_CASE_						 = os.path.join(_lowercase       ,       '''vocab.txt'''    )
											with open(_lowercase       ,       '''w'''       ,       encoding='''utf-8'''    ) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]    )    )
											SCREAMING_SNAKE_CASE_						 = CustomTokenizer(_lowercase    )

									# No fast custom tokenizer
									tokenizer.push_to_hub('''test-dynamic-tokenizer'''       ,       use_auth_token=self._token    )

									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'       ,       trust_remote_code=_lowercase    )
									# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__       ,       '''CustomTokenizer'''    )

									# Fast and slow custom tokenizer
									CustomTokenizerFast.register_for_auto_class()
									with tempfile.TemporaryDirectory() as tmp_dir:
											SCREAMING_SNAKE_CASE_						 = os.path.join(_lowercase       ,       '''vocab.txt'''    )
											with open(_lowercase       ,       '''w'''       ,       encoding='''utf-8'''    ) as vocab_writer:
													vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]    )    )

											SCREAMING_SNAKE_CASE_						 = BertTokenizerFast.from_pretrained(_lowercase    )
											bert_tokenizer.save_pretrained(_lowercase    )
											SCREAMING_SNAKE_CASE_						 = CustomTokenizerFast.from_pretrained(_lowercase    )

									tokenizer.push_to_hub('''test-dynamic-tokenizer'''       ,       use_auth_token=self._token    )

									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(f'{USER}/test-dynamic-tokenizer'       ,       trust_remote_code=_lowercase    )
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__       ,       '''CustomTokenizerFast'''    )
									SCREAMING_SNAKE_CASE_						 = AutoTokenizer.from_pretrained(
									    f'{USER}/test-dynamic-tokenizer'       ,       use_fast=_lowercase       ,       trust_remote_code=_lowercase    )
									# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
									self.assertEqual(tokenizer.__class__.__name__       ,       '''CustomTokenizer'''    )







class  snake_case     (							unittest.TestCase   ):

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = Trie()
									trie.add('''Hello 友達'''    )
									self.assertEqual(trie.data       ,       {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}    )
									trie.add('''Hello'''    )
									trie.data
									self.assertEqual(trie.data       ,       {'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = Trie()
									self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''    )       ,       ['''[CLS] This is a extra_id_100''']    )
									trie.add('''[CLS]'''    )
									trie.add('''extra_id_1'''    )
									trie.add('''extra_id_100'''    )
									self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''    )       ,       ['''[CLS]''', ''' This is a ''', '''extra_id_100''']    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = Trie()
									trie.add('''A'''    )
									self.assertEqual(trie.split('''ABC'''    )       ,       ['''A''', '''BC''']    )
									self.assertEqual(trie.split('''BCA'''    )       ,       ['''BC''', '''A''']    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = Trie()
									trie.add('''TOKEN]'''    )
									trie.add('''[SPECIAL_TOKEN]'''    )
									self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''    )       ,       ['''This is something ''', '''[SPECIAL_TOKEN]''']    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = Trie()
									trie.add('''A'''    )
									trie.add('''P'''    )
									trie.add('''[SPECIAL_TOKEN]'''    )
									self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''    )       ,       ['''This is something ''', '''[SPECIAL_TOKEN]''']    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = Trie()
									trie.add('''AB'''    )
									trie.add('''B'''    )
									trie.add('''C'''    )
									self.assertEqual(trie.split('''ABC'''    )       ,       ['''AB''', '''C''']    )

							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = Trie()
									trie.add('''ABC'''    )
									trie.add('''B'''    )
									trie.add('''CD'''    )
									self.assertEqual(trie.split('''ABCD'''    )       ,       ['''ABC''', '''D''']    )







							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = Trie()
									SCREAMING_SNAKE_CASE_						 = trie.cut_text('''ABC'''       ,       [0, 0, 2, 1, 2, 3]    )
									self.assertEqual(_lowercase       ,       ['''AB''', '''C''']    )

"""simple docstring"""




import copy
from typing import Dict, Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING
from ..detr import DetrConfig
from ..swin import SwinConfig


lowerCAmelCase__							=				{
    'facebook/maskformer-swin-base-ade': (
        'https://huggingface.co/facebook/maskformer-swin-base-ade/blob/main/config.json'
    )
    # See all MaskFormer models at https://huggingface.co/models?filter=maskformer
}

lowerCAmelCase__							=				logging.get_logger(__name__)





class  snake_case     (							__lowercase   ):
							UpperCAmelCase__						     =      '''maskformer'''
							UpperCAmelCase__						     =      {'''hidden_size''': '''mask_feature_size'''}
							UpperCAmelCase__						     =      ['''resnet''', '''swin''']
							UpperCAmelCase__						     =      ['''detr''']

							def __init__(self       ,       SCREAMING_SNAKE_CASE_ = 2_56       ,       SCREAMING_SNAKE_CASE_ = 2_56       ,       SCREAMING_SNAKE_CASE_ = 0.1       ,       SCREAMING_SNAKE_CASE_ = False       ,       SCREAMING_SNAKE_CASE_ = None       ,       SCREAMING_SNAKE_CASE_ = None       ,       SCREAMING_SNAKE_CASE_ = 0.02       ,       SCREAMING_SNAKE_CASE_ = 1.0       ,       SCREAMING_SNAKE_CASE_ = 1.0       ,       SCREAMING_SNAKE_CASE_ = 1.0       ,       SCREAMING_SNAKE_CASE_ = 20.0       ,       SCREAMING_SNAKE_CASE_ = None       ,       **SCREAMING_SNAKE_CASE_       ,       ):

									"""simple docstring"""







									if backbone_config is None:
											# fall back to https://huggingface.co/microsoft/swin-base-patch4-window12-384-in22k
											SCREAMING_SNAKE_CASE_						 = SwinConfig(
											    image_size=3_84       ,       in_channels=3       ,       patch_size=4       ,       embed_dim=1_28       ,       depths=[2, 2, 18, 2]       ,       num_heads=[4, 8, 16, 32]       ,       window_size=12       ,       drop_path_rate=0.3       ,       out_features=['''stage1''', '''stage2''', '''stage3''', '''stage4''']       ,       )

									if isinstance(SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):
											SCREAMING_SNAKE_CASE_						 = backbone_config.pop('''model_type'''    )
											SCREAMING_SNAKE_CASE_						 = CONFIG_MAPPING[backbone_model_type]
											SCREAMING_SNAKE_CASE_						 = config_class.from_dict(SCREAMING_SNAKE_CASE_    )

									# verify that the backbone is supported
									if backbone_config.model_type not in self.backbones_supported:
											logger.warning_once(
											    f'Backbone {backbone_config.model_type} is not a supported model and may not be compatible with MaskFormer. '
											    f'Supported model types: {",".join(self.backbones_supported    )}'    )

									if decoder_config is None:
											# fall back to https://huggingface.co/facebook/detr-resnet-50
											SCREAMING_SNAKE_CASE_						 = DetrConfig()
									else:
											# verify that the decoder is supported
											SCREAMING_SNAKE_CASE_						 = (
											    decoder_config.pop('''model_type'''    ) if isinstance(SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ) else decoder_config.model_type
											)
											if decoder_type not in self.decoders_supported:
													raise ValueError(
													    f'Transformer Decoder {decoder_type} not supported, please use one of'
													    f' {",".join(self.decoders_supported    )}'    )
											if isinstance(SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_    ):
													SCREAMING_SNAKE_CASE_						 = CONFIG_MAPPING[decoder_type]
													SCREAMING_SNAKE_CASE_						 = config_class.from_dict(SCREAMING_SNAKE_CASE_    )

									SCREAMING_SNAKE_CASE_						 = backbone_config
									SCREAMING_SNAKE_CASE_						 = decoder_config
									# main feature dimension for the model
									SCREAMING_SNAKE_CASE_						 = fpn_feature_size
									SCREAMING_SNAKE_CASE_						 = mask_feature_size
									# initializer
									SCREAMING_SNAKE_CASE_						 = init_std
									SCREAMING_SNAKE_CASE_						 = init_xavier_std
									# Hungarian matcher && loss
									SCREAMING_SNAKE_CASE_						 = cross_entropy_weight
									SCREAMING_SNAKE_CASE_						 = dice_weight
									SCREAMING_SNAKE_CASE_						 = mask_weight
									SCREAMING_SNAKE_CASE_						 = use_auxiliary_loss
									SCREAMING_SNAKE_CASE_						 = no_object_weight
									SCREAMING_SNAKE_CASE_						 = output_auxiliary_logits

									SCREAMING_SNAKE_CASE_						 = self.decoder_config.encoder_attention_heads
									SCREAMING_SNAKE_CASE_						 = self.decoder_config.num_hidden_layers
									super().__init__(**SCREAMING_SNAKE_CASE_    )

							@classmethod
							def _lowercase					(cls       ,       SCREAMING_SNAKE_CASE_       ,       SCREAMING_SNAKE_CASE_       ,       **SCREAMING_SNAKE_CASE_    ):

									"""simple docstring"""







									return cls(
									    backbone_config=SCREAMING_SNAKE_CASE_       ,       decoder_config=SCREAMING_SNAKE_CASE_       ,       **SCREAMING_SNAKE_CASE_       ,       )







							def _lowercase					(self    ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_						 = copy.deepcopy(self.__dict__    )
									SCREAMING_SNAKE_CASE_						 = self.backbone_config.to_dict()
									SCREAMING_SNAKE_CASE_						 = self.decoder_config.to_dict()
									SCREAMING_SNAKE_CASE_						 = self.__class__.model_type
									return output