


import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast







class 				snake_case__(							unittest.TestCase							):







						'''simple docstring'''




						def 						lowercase_ (				self							)					->    int:
										lowerCAmelCase_ :      str    				=				0




						@slow
						def 						lowercase_ (				self							)					->    Optional[Any]:
										for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
														lowerCAmelCase_ :      int    				=				AutoTokenizer.from_pretrained(__lowercase							)
														self.assertIsNotNone(__lowercase							)
														self.assertIsInstance(__lowercase   ,						(BertTokenizer, BertTokenizerFast)							)
														self.assertGreater(len(__lowercase							)   ,						0							)

										for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
														lowerCAmelCase_ :      Optional[int]    				=				AutoTokenizer.from_pretrained(__lowercase							)
														self.assertIsNotNone(__lowercase							)
														self.assertIsInstance(__lowercase   ,						(GPTaTokenizer, GPTaTokenizerFast)							)
														self.assertGreater(len(__lowercase							)   ,						0							)




						def 						lowercase_ (				self							)					->    List[Any]:
										lowerCAmelCase_ :      Optional[Any]    				=				AutoTokenizer.from_pretrained(__lowercase							)
										self.assertIsInstance(__lowercase   ,						(BertTokenizer, BertTokenizerFast)							)
										self.assertEqual(tokenizer.vocab_size   ,						1_2							)




						def 						lowercase_ (				self							)					->    int:
										lowerCAmelCase_ :      int    				=				AutoTokenizer.from_pretrained(__lowercase							)
										self.assertIsInstance(__lowercase   ,						(RobertaTokenizer, RobertaTokenizerFast)							)
										self.assertEqual(tokenizer.vocab_size   ,						2_0							)




						def 						lowercase_ (				self							)					->    Union[str, Any]:
										lowerCAmelCase_ :      Optional[Any]    				=				AutoConfig.from_pretrained(__lowercase							)
										self.assertIsInstance(__lowercase   ,						__lowercase							)
										# Check that tokenizer_type ≠ model_type
										lowerCAmelCase_ :      Dict    				=				AutoTokenizer.from_pretrained(__lowercase   ,						config=__lowercase							)
										self.assertIsInstance(__lowercase   ,						(BertTokenizer, BertTokenizerFast)							)
										self.assertEqual(tokenizer.vocab_size   ,						1_2							)




						def 						lowercase_ (				self							)					->    int:
										with tempfile.TemporaryDirectory() as tmp_dir:
														shutil.copy('''./tests/fixtures/vocab.txt'''   ,						os.path.join(__lowercase   ,						'''vocab.txt'''							)							)

														lowerCAmelCase_ :      Any    				=				AutoTokenizer.from_pretrained(__lowercase   ,						tokenizer_type='''bert'''   ,						use_fast=__lowercase							)
														self.assertIsInstance(__lowercase   ,						__lowercase							)

										with tempfile.TemporaryDirectory() as tmp_dir:
														shutil.copy('''./tests/fixtures/vocab.json'''   ,						os.path.join(__lowercase   ,						'''vocab.json'''							)							)
														shutil.copy('''./tests/fixtures/merges.txt'''   ,						os.path.join(__lowercase   ,						'''merges.txt'''							)							)

														lowerCAmelCase_ :      Union[str, Any]    				=				AutoTokenizer.from_pretrained(__lowercase   ,						tokenizer_type='''gpt2'''   ,						use_fast=__lowercase							)
														self.assertIsInstance(__lowercase   ,						__lowercase							)




						@require_tokenizers
						def 						lowercase_ (				self							)					->    Union[str, Any]:
										with tempfile.TemporaryDirectory() as tmp_dir:
														shutil.copy('''./tests/fixtures/vocab.txt'''   ,						os.path.join(__lowercase   ,						'''vocab.txt'''							)							)

														lowerCAmelCase_ :      str    				=				AutoTokenizer.from_pretrained(__lowercase   ,						tokenizer_type='''bert'''							)
														self.assertIsInstance(__lowercase   ,						__lowercase							)

										with tempfile.TemporaryDirectory() as tmp_dir:
														shutil.copy('''./tests/fixtures/vocab.json'''   ,						os.path.join(__lowercase   ,						'''vocab.json'''							)							)
														shutil.copy('''./tests/fixtures/merges.txt'''   ,						os.path.join(__lowercase   ,						'''merges.txt'''							)							)

														lowerCAmelCase_ :      str    				=				AutoTokenizer.from_pretrained(__lowercase   ,						tokenizer_type='''gpt2'''							)
														self.assertIsInstance(__lowercase   ,						__lowercase							)




						def 						lowercase_ (				self							)					->    Dict:
										with pytest.raises(__lowercase							):
														AutoTokenizer.from_pretrained('''./'''   ,						tokenizer_type='''xxx'''							)




						@require_tokenizers
						def 						lowercase_ (				self							)					->    int:
										for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
														lowerCAmelCase_ :      List[Any]    				=				tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased'''							)
														self.assertIsInstance(__lowercase   ,						(BertTokenizer, BertTokenizerFast)							)

														if isinstance(__lowercase   ,						__lowercase							):
																		self.assertEqual(tokenizer.basic_tokenizer.do_lower_case   ,						__lowercase							)
														else:
																		self.assertEqual(tokenizer.do_lower_case   ,						__lowercase							)

														self.assertEqual(tokenizer.model_max_length   ,						5_1_2							)




						@require_tokenizers
						def 						lowercase_ (				self							)					->    Optional[int]:
										for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
														with self.assertRaisesRegex(
														    __lowercase   ,						'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''   ,						):
																		lowerCAmelCase_ :      Any    				=				tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists'''							)




						def 						lowercase_ (				self							)					->    List[Any]:
										# tests: https://github.com/huggingface/transformers/pull/13251
										# 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
										# 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
										lowerCAmelCase_ :      Optional[int]    				=				TOKENIZER_MAPPING.values()
										lowerCAmelCase_ :      List[Any]    				=				[]

										for slow_tok, fast_tok in tokenizers:
														if slow_tok is not None:
																		tokenizer_names.append(slow_tok.__name__							)

														if fast_tok is not None:
																		tokenizer_names.append(fast_tok.__name__							)

										for tokenizer_name in tokenizer_names:
														# must find the right class
														tokenizer_class_from_name(__lowercase							)




						@require_tokenizers
						def 						lowercase_ (				self							)					->    Dict:
										self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''   ,						use_fast=__lowercase							)   ,						__lowercase							)
										self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''							)   ,						__lowercase							)




						@require_tokenizers
						def 						lowercase_ (				self							)					->    int:
										lowerCAmelCase_ :      Optional[int]    				=				AutoTokenizer.from_pretrained('''distilbert-base-uncased'''   ,						do_lower_case=__lowercase							)
										lowerCAmelCase_ :      Optional[Any]    				=				'''Hello, world. How are you?'''
										lowerCAmelCase_ :      List[str]    				=				tokenizer.tokenize(__lowercase							)
										self.assertEqual('''[UNK]'''   ,						tokens[0]							)

										lowerCAmelCase_ :      Optional[Any]    				=				AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''   ,						do_lower_case=__lowercase							)
										lowerCAmelCase_ :      List[str]    				=				tokenizer.tokenize(__lowercase							)
										self.assertEqual('''[UNK]'''   ,						tokens[0]							)




						@require_tokenizers
						def 						lowercase_ (				self							)					->    Any:
										lowerCAmelCase_ :      List[str]    				=				AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config'''							)
										self.assertEqual(type(__lowercase							)   ,						__lowercase							)
										self.assertEqual(tokenizer.model_max_length   ,						5_1_2							)
										self.assertEqual(tokenizer.vocab_size   ,						3_0_0_0_0							)
										self.assertEqual(tokenizer.unk_token   ,						'''[UNK]'''							)
										self.assertEqual(tokenizer.padding_side   ,						'''right'''							)
										self.assertEqual(tokenizer.truncation_side   ,						'''right'''							)




						def 						lowercase_ (				self							)					->    Tuple:
										lowerCAmelCase_ :      Tuple    				=				AutoTokenizer.from_pretrained(__lowercase							)
										self.assertIsInstance(__lowercase   ,						(BertTokenizer, BertTokenizerFast)							)
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(__lowercase							)
														lowerCAmelCase_ :      str    				=				AutoTokenizer.from_pretrained(__lowercase							)

										self.assertIsInstance(__lowercase   ,						tokenizer.__class__							)
										self.assertEqual(tokenizera.vocab_size   ,						1_2							)




						def 						lowercase_ (				self							)					->    List[str]:
										lowerCAmelCase_ :      int    				=				AutoTokenizer.from_pretrained('''ctrl'''							)
										# There is no fast CTRL so this always gives us a slow tokenizer.
										self.assertIsInstance(__lowercase   ,						__lowercase							)




						def 						lowercase_ (				self							)					->    Optional[Any]:
										# Check we can load the tokenizer config of an online model.
										lowerCAmelCase_ :      Union[str, Any]    				=				get_tokenizer_config('''bert-base-cased'''							)
										lowerCAmelCase_ :      Union[str, Any]    				=				config.pop('''_commit_hash'''   ,						__lowercase							)
										# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
										self.assertEqual(__lowercase   ,						{'''do_lower_case''': False}							)

										# This model does not have a tokenizer_config so we get back an empty dict.
										lowerCAmelCase_ :      List[str]    				=				get_tokenizer_config(__lowercase							)
										self.assertDictEqual(__lowercase   ,						{}							)

										# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
										lowerCAmelCase_ :      Dict    				=				AutoTokenizer.from_pretrained(__lowercase							)
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(__lowercase							)
														lowerCAmelCase_ :      List[str]    				=				get_tokenizer_config(__lowercase							)

										# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
										self.assertEqual(config['''tokenizer_class''']   ,						'''BertTokenizer'''							)




						def 						lowercase_ (				self							)					->    Union[str, Any]:
										try:
														AutoConfig.register('''custom'''   ,						__lowercase							)

														AutoTokenizer.register(__lowercase   ,						slow_tokenizer_class=__lowercase							)
														# Trying to register something existing in the Transformers library will raise an error
														with self.assertRaises(__lowercase							):
																		AutoTokenizer.register(__lowercase   ,						slow_tokenizer_class=__lowercase							)

														lowerCAmelCase_ :      Optional[Any]    				=				CustomTokenizer.from_pretrained(__lowercase							)
														with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(__lowercase							)

																		lowerCAmelCase_ :      List[str]    				=				AutoTokenizer.from_pretrained(__lowercase							)
																		self.assertIsInstance(__lowercase   ,						__lowercase							)

										finally:
														if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
														if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]




						@require_tokenizers
						def 						lowercase_ (				self							)					->    List[str]:
										try:
														AutoConfig.register('''custom'''   ,						__lowercase							)

														# Can register in two steps
														AutoTokenizer.register(__lowercase   ,						slow_tokenizer_class=__lowercase							)
														self.assertEqual(TOKENIZER_MAPPING[CustomConfig]   ,						(CustomTokenizer, None)							)
														AutoTokenizer.register(__lowercase   ,						fast_tokenizer_class=__lowercase							)
														self.assertEqual(TOKENIZER_MAPPING[CustomConfig]   ,						(CustomTokenizer, CustomTokenizerFast)							)

														del TOKENIZER_MAPPING._extra_content[CustomConfig]
														# Can register in one step
														AutoTokenizer.register(
														    __lowercase   ,						slow_tokenizer_class=__lowercase   ,						fast_tokenizer_class=__lowercase							)
														self.assertEqual(TOKENIZER_MAPPING[CustomConfig]   ,						(CustomTokenizer, CustomTokenizerFast)							)

														# Trying to register something existing in the Transformers library will raise an error
														with self.assertRaises(__lowercase							):
																		AutoTokenizer.register(__lowercase   ,						fast_tokenizer_class=__lowercase							)

														# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
														# and that model does not have a tokenizer.json
														with tempfile.TemporaryDirectory() as tmp_dir:
																		lowerCAmelCase_ :      Union[str, Any]    				=				BertTokenizerFast.from_pretrained(__lowercase							)
																		bert_tokenizer.save_pretrained(__lowercase							)
																		lowerCAmelCase_ :      List[Any]    				=				CustomTokenizerFast.from_pretrained(__lowercase							)

														with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(__lowercase							)

																		lowerCAmelCase_ :      Dict    				=				AutoTokenizer.from_pretrained(__lowercase							)
																		self.assertIsInstance(__lowercase   ,						__lowercase							)

																		lowerCAmelCase_ :      str    				=				AutoTokenizer.from_pretrained(__lowercase   ,						use_fast=__lowercase							)
																		self.assertIsInstance(__lowercase   ,						__lowercase							)

										finally:
														if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
														if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]




						def 						lowercase_ (				self							)					->    int:
										# If remote code is not set, we will time out when asking whether to load the model.
										with self.assertRaises(__lowercase							):
														lowerCAmelCase_ :      Optional[int]    				=				AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''							)
										# If remote code is disabled, we can't load this config.
										with self.assertRaises(__lowercase							):
														lowerCAmelCase_ :      int    				=				AutoTokenizer.from_pretrained(
														    '''hf-internal-testing/test_dynamic_tokenizer'''   ,						trust_remote_code=__lowercase							)

										lowerCAmelCase_ :      Optional[Any]    				=				AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''   ,						trust_remote_code=__lowercase							)
										self.assertTrue(tokenizer.special_attribute_present							)
										# Test tokenizer can be reloaded.
										with tempfile.TemporaryDirectory() as tmp_dir:
														tokenizer.save_pretrained(__lowercase							)
														lowerCAmelCase_ :      List[Any]    				=				AutoTokenizer.from_pretrained(__lowercase   ,						trust_remote_code=__lowercase							)
										self.assertTrue(reloaded_tokenizer.special_attribute_present							)

										if is_tokenizers_available():
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizerFast'''							)
														self.assertEqual(reloaded_tokenizer.__class__.__name__   ,						'''NewTokenizerFast'''							)

														# Test we can also load the slow version
														lowerCAmelCase_ :      int    				=				AutoTokenizer.from_pretrained(
														    '''hf-internal-testing/test_dynamic_tokenizer'''   ,						trust_remote_code=__lowercase   ,						use_fast=__lowercase							)
														self.assertTrue(tokenizer.special_attribute_present							)
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizer'''							)
														# Test tokenizer can be reloaded.
														with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer.save_pretrained(__lowercase							)
																		lowerCAmelCase_ :      Union[str, Any]    				=				AutoTokenizer.from_pretrained(__lowercase   ,						trust_remote_code=__lowercase   ,						use_fast=__lowercase							)
														self.assertEqual(reloaded_tokenizer.__class__.__name__   ,						'''NewTokenizer'''							)
														self.assertTrue(reloaded_tokenizer.special_attribute_present							)
										else:
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizer'''							)
														self.assertEqual(reloaded_tokenizer.__class__.__name__   ,						'''NewTokenizer'''							)




						@require_tokenizers
						def 						lowercase_ (				self							)					->    Tuple:







										class 				snake_case__(							UpperCAmelCase__							):







																'''simple docstring'''




																SCREAMING_SNAKE_CASE__      :  Optional[int]					=							False







										class 				snake_case__(							UpperCAmelCase__							):







																'''simple docstring'''




																SCREAMING_SNAKE_CASE__      :  Optional[int]					=							NewTokenizer
																SCREAMING_SNAKE_CASE__      :  Optional[int]					=							False

										try:
														AutoConfig.register('''custom'''   ,						__lowercase							)
														AutoTokenizer.register(__lowercase   ,						slow_tokenizer_class=__lowercase							)
														AutoTokenizer.register(__lowercase   ,						fast_tokenizer_class=__lowercase							)
														# If remote code is not set, the default is to use local
														lowerCAmelCase_ :      List[Any]    				=				AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''							)
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizerFast'''							)
														self.assertFalse(tokenizer.special_attribute_present							)
														lowerCAmelCase_ :      Union[str, Any]    				=				AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''   ,						use_fast=__lowercase							)
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizer'''							)
														self.assertFalse(tokenizer.special_attribute_present							)

														# If remote code is disabled, we load the local one.
														lowerCAmelCase_ :      str    				=				AutoTokenizer.from_pretrained(
														    '''hf-internal-testing/test_dynamic_tokenizer'''   ,						trust_remote_code=__lowercase							)
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizerFast'''							)
														self.assertFalse(tokenizer.special_attribute_present							)
														lowerCAmelCase_ :      Union[str, Any]    				=				AutoTokenizer.from_pretrained(
														    '''hf-internal-testing/test_dynamic_tokenizer'''   ,						trust_remote_code=__lowercase   ,						use_fast=__lowercase							)
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizer'''							)
														self.assertFalse(tokenizer.special_attribute_present							)

														# If remote is enabled, we load from the Hub
														lowerCAmelCase_ :      Any    				=				AutoTokenizer.from_pretrained(
														    '''hf-internal-testing/test_dynamic_tokenizer'''   ,						trust_remote_code=__lowercase							)
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizerFast'''							)
														self.assertTrue(tokenizer.special_attribute_present							)
														lowerCAmelCase_ :      Dict    				=				AutoTokenizer.from_pretrained(
														    '''hf-internal-testing/test_dynamic_tokenizer'''   ,						trust_remote_code=__lowercase   ,						use_fast=__lowercase							)
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizer'''							)
														self.assertTrue(tokenizer.special_attribute_present							)

										finally:
														if "custom" in CONFIG_MAPPING._extra_content:
																		del CONFIG_MAPPING._extra_content["custom"]
														if CustomConfig in TOKENIZER_MAPPING._extra_content:
																		del TOKENIZER_MAPPING._extra_content[CustomConfig]




						def 						lowercase_ (				self							)					->    Union[str, Any]:
										lowerCAmelCase_ :      str    				=				AutoTokenizer.from_pretrained(
										    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''   ,						trust_remote_code=__lowercase							)
										self.assertTrue(tokenizer.special_attribute_present							)
										if is_tokenizers_available():
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizerFast'''							)

														# Test we can also load the slow version
														lowerCAmelCase_ :      Optional[Any]    				=				AutoTokenizer.from_pretrained(
														    '''hf-internal-testing/test_dynamic_tokenizer_legacy'''   ,						trust_remote_code=__lowercase   ,						use_fast=__lowercase							)
														self.assertTrue(tokenizer.special_attribute_present							)
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizer'''							)
										else:
														self.assertEqual(tokenizer.__class__.__name__   ,						'''NewTokenizer'''							)




						def 						lowercase_ (				self							)					->    int:
										with self.assertRaisesRegex(
										    __lowercase   ,						'''bert-base is not a local folder and is not a valid model identifier'''							):
														lowerCAmelCase_ :      Union[str, Any]    				=				AutoTokenizer.from_pretrained('''bert-base'''							)




						def 						lowercase_ (				self							)					->    Optional[int]:
										with self.assertRaisesRegex(
										    __lowercase   ,						R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''							):
														lowerCAmelCase_ :      Union[str, Any]    				=				AutoTokenizer.from_pretrained(__lowercase   ,						revision='''aaaaaa'''							)




						def 						lowercase_ (				self							)					->    str:
										# Make sure we have cached the tokenizer.
										lowerCAmelCase_ :      Any    				=				AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''							)
										with RequestCounter() as counter:
														lowerCAmelCase_ :      Optional[Any]    				=				AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''							)
														self.assertEqual(counter.get_request_count   ,						0							)
														self.assertEqual(counter.head_request_count   ,						1							)
														self.assertEqual(counter.other_request_count   ,						0							)



import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import DetrConfig, MaskFormerConfig, SwinConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import MaskFormerForInstanceSegmentation, MaskFormerModel

						if is_vision_available():
												from transformers import MaskFormerImageProcessor

if is_vision_available():
						from PIL import Image







class 				snake_case__:







						'''simple docstring'''




						def __init__(				self   ,						__lowercase   ,						__lowercase=2   ,						__lowercase=True   ,						__lowercase=False   ,						__lowercase=1_0   ,						__lowercase=3   ,						__lowercase=3_2 * 4   ,						__lowercase=3_2 * 6   ,						__lowercase=4   ,						__lowercase=3_2   ,						)					->    Union[str, Any]:
										lowerCAmelCase_ :      str    				=				parent
										lowerCAmelCase_ :      Optional[Any]    				=				batch_size
										lowerCAmelCase_ :      List[Any]    				=				is_training
										lowerCAmelCase_ :      Optional[Any]    				=				use_auxiliary_loss
										lowerCAmelCase_ :      List[Any]    				=				num_queries
										lowerCAmelCase_ :      str    				=				num_channels
										lowerCAmelCase_ :      Dict    				=				min_size
										lowerCAmelCase_ :      List[str]    				=				max_size
										lowerCAmelCase_ :      Any    				=				num_labels
										lowerCAmelCase_ :      str    				=				mask_feature_size




						def 						lowercase_ (				self							)					->    List[Any]:
										lowerCAmelCase_ :      Optional[Any]    				=				floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size]							).to(
										    __lowercase							)

										lowerCAmelCase_ :      Optional[Any]    				=				torch.ones([self.batch_size, self.min_size, self.max_size]   ,						device=__lowercase							)

										lowerCAmelCase_ :      Union[str, Any]    				=				(
										    torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size]   ,						device=__lowercase							) > 0.5
										).float()
										lowerCAmelCase_ :      List[str]    				=				(torch.rand((self.batch_size, self.num_labels)   ,						device=__lowercase							) > 0.5).long()

										lowerCAmelCase_ :      Dict    				=				self.get_config()
										return config, pixel_values, pixel_mask, mask_labels, class_labels




						def 						lowercase_ (				self							)					->    List[str]:
										return MaskFormerConfig.from_backbone_and_decoder_configs(
										    backbone_config=SwinConfig(
										        depths=[1, 1, 1, 1]   ,						)   ,						decoder_config=DetrConfig(
										        decoder_ffn_dim=1_2_8   ,						num_queries=self.num_queries   ,						decoder_attention_heads=2   ,						d_model=self.mask_feature_size   ,						)   ,						mask_feature_size=self.mask_feature_size   ,						fpn_feature_size=self.mask_feature_size   ,						num_channels=self.num_channels   ,						num_labels=self.num_labels   ,						)




						def 						lowercase_ (				self							)					->    Union[str, Any]:
										lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_ :      int    				=				self.prepare_config_and_inputs()
										lowerCAmelCase_ :      Union[str, Any]    				=				{'''pixel_values''': pixel_values, '''pixel_mask''': pixel_mask}
										return config, inputs_dict




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase							)					->    Any:
										lowerCAmelCase_ :      Optional[int]    				=				output.encoder_hidden_states
										lowerCAmelCase_ :      List[Any]    				=				output.pixel_decoder_hidden_states
										lowerCAmelCase_ :      Optional[Any]    				=				output.transformer_decoder_hidden_states

										self.parent.assertTrue(len(__lowercase							)   ,						len(config.backbone_config.depths							)							)
										self.parent.assertTrue(len(__lowercase							)   ,						len(config.backbone_config.depths							)							)
										self.parent.assertTrue(len(__lowercase							)   ,						config.decoder_config.decoder_layers							)




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase   ,						__lowercase   ,						__lowercase=False							)					->    int:
										with torch.no_grad():
														lowerCAmelCase_ :      List[Any]    				=				MaskFormerModel(config=__lowercase							)
														model.to(__lowercase							)
														model.eval()

														lowerCAmelCase_ :      Optional[Any]    				=				model(pixel_values=__lowercase   ,						pixel_mask=__lowercase							)
														lowerCAmelCase_ :      Optional[int]    				=				model(__lowercase   ,						output_hidden_states=__lowercase							)
										# the correct shape of output.transformer_decoder_hidden_states ensure the correcteness of the
										# encoder and pixel decoder
										self.parent.assertEqual(
										    output.transformer_decoder_last_hidden_state.shape   ,						(self.batch_size, self.num_queries, self.mask_feature_size)   ,						)
										# let's ensure the other two hidden state exists
										self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None							)
										self.parent.assertTrue(output.encoder_last_hidden_state is not None							)

										if output_hidden_states:
														self.check_output_hidden_state(__lowercase   ,						__lowercase							)




						def 						lowercase_ (				self   ,						__lowercase   ,						__lowercase   ,						__lowercase   ,						__lowercase   ,						__lowercase							)					->    Any:
										lowerCAmelCase_ :      Any    				=				MaskFormerForInstanceSegmentation(config=__lowercase							)
										model.to(__lowercase							)
										model.eval()

										def comm_check_on_output(__lowercase							):
														# let's still check that all the required stuff is there
														self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None							)
														self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None							)
														self.parent.assertTrue(result.encoder_last_hidden_state is not None							)
														# okay, now we need to check the logits shape
														# due to the encoder compression, masks have a //4 spatial size
														self.parent.assertEqual(
														    result.masks_queries_logits.shape   ,						(self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4)   ,						)
														# + 1 for null class
														self.parent.assertEqual(
														    result.class_queries_logits.shape   ,						(self.batch_size, self.num_queries, self.num_labels + 1)							)

										with torch.no_grad():
														lowerCAmelCase_ :      int    				=				model(pixel_values=__lowercase   ,						pixel_mask=__lowercase							)
														lowerCAmelCase_ :      Any    				=				model(__lowercase							)

														comm_check_on_output(__lowercase							)

														lowerCAmelCase_ :      List[Any]    				=				model(
														    pixel_values=__lowercase   ,						pixel_mask=__lowercase   ,						mask_labels=__lowercase   ,						class_labels=__lowercase							)

										comm_check_on_output(__lowercase							)

										self.parent.assertTrue(result.loss is not None							)
										self.parent.assertEqual(result.loss.shape   ,						torch.Size([1]							)							)







@require_torch
class 				snake_case__(							UpperCAmelCase__,						UpperCAmelCase__,						unittest.TestCase							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__      :  List[str]					=							(MaskFormerModel, MaskFormerForInstanceSegmentation) if is_torch_available() else ()
						SCREAMING_SNAKE_CASE__      :  Tuple					=							(
						    {"""feature-extraction""": MaskFormerModel, """image-segmentation""": MaskFormerForInstanceSegmentation}
						    if is_torch_available()
						    else {}
						)

						SCREAMING_SNAKE_CASE__      :  Tuple					=							False
						SCREAMING_SNAKE_CASE__      :  Dict					=							False
						SCREAMING_SNAKE_CASE__      :  Tuple					=							False
						SCREAMING_SNAKE_CASE__      :  List[str]					=							False




						def 						lowercase_ (				self							)					->    List[Any]:
										lowerCAmelCase_ :      Any    				=				MaskFormerModelTester(self							)
										lowerCAmelCase_ :      str    				=				ConfigTester(self   ,						config_class=__lowercase   ,						has_text_modality=__lowercase							)




						def 						lowercase_ (				self							)					->    Any:
										self.config_tester.run_common_tests()




						def 						lowercase_ (				self							)					->    List[str]:
										lowerCAmelCase_				,  lowerCAmelCase_ :      Any    				=				self.model_tester.prepare_config_and_inputs_for_common()
										self.model_tester.create_and_check_maskformer_model(__lowercase   ,						**__lowercase   ,						output_hidden_states=__lowercase							)




						def 						lowercase_ (				self							)					->    Optional[Any]:
										lowerCAmelCase_ :      Optional[Any]    				=				self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_maskformer_instance_segmentation_head_model(*__lowercase							)




						@unittest.skip(reason='''MaskFormer does not use inputs_embeds'''							)
						def 						lowercase_ (				self							)					->    str:
										pass




						@unittest.skip(reason='''MaskFormer does not have a get_input_embeddings method'''							)
						def 						lowercase_ (				self							)					->    Optional[Any]:
										pass




						@unittest.skip(reason='''MaskFormer is not a generative model'''							)
						def 						lowercase_ (				self							)					->    Optional[Any]:
										pass




						@unittest.skip(reason='''MaskFormer does not use token embeddings'''							)
						def 						lowercase_ (				self							)					->    Union[str, Any]:
										pass




						@require_torch_multi_gpu
						@unittest.skip(
						    reason='''MaskFormer has some layers using `add_module` which doesn\'t work well with `nn.DataParallel`'''							)
						def 						lowercase_ (				self							)					->    Optional[Any]:
										pass




						@unittest.skip('''Will be fixed soon by reducing the size of the model used for common tests.'''							)
						def 						lowercase_ (				self							)					->    Dict:
										pass




						def 						lowercase_ (				self							)					->    List[str]:
										lowerCAmelCase_				,  lowerCAmelCase_ :      str    				=				self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
														lowerCAmelCase_ :      Tuple    				=				model_class(__lowercase							)
														lowerCAmelCase_ :      Dict    				=				inspect.signature(model.forward							)
														# signature.parameters is an OrderedDict => so arg_names order is deterministic
														lowerCAmelCase_ :      str    				=				[*signature.parameters.keys()]

														lowerCAmelCase_ :      Tuple    				=				['''pixel_values''']
														self.assertListEqual(arg_names[:1]   ,						__lowercase							)




						@slow
						def 						lowercase_ (				self							)					->    Optional[int]:
										for model_name in ["facebook/maskformer-swin-small-coco"]:
														lowerCAmelCase_ :      str    				=				MaskFormerModel.from_pretrained(__lowercase							)
														self.assertIsNotNone(__lowercase							)




						def 						lowercase_ (				self							)					->    List[Any]:
										lowerCAmelCase_ :      Tuple    				=				(self.model_tester.min_size,) * 2
										lowerCAmelCase_ :      List[Any]    				=				{
										    '''pixel_values''': torch.randn((2, 3, *size)   ,						device=__lowercase							),
										    '''mask_labels''': torch.randn((2, 1_0, *size)   ,						device=__lowercase							),
										    '''class_labels''': torch.zeros(2   ,						1_0   ,						device=__lowercase							).long(),
										}

										lowerCAmelCase_ :      Tuple    				=				MaskFormerForInstanceSegmentation(MaskFormerConfig()							).to(__lowercase							)
										lowerCAmelCase_ :      Dict    				=				model(**__lowercase							)
										self.assertTrue(outputs.loss is not None							)




						def 						lowercase_ (				self							)					->    Dict:
										lowerCAmelCase_				,  lowerCAmelCase_ :      List[Any]    				=				self.model_tester.prepare_config_and_inputs_for_common()
										self.model_tester.create_and_check_maskformer_model(__lowercase   ,						**__lowercase   ,						output_hidden_states=__lowercase							)




						def 						lowercase_ (				self							)					->    int:
										lowerCAmelCase_				,  lowerCAmelCase_ :      Tuple    				=				self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
														lowerCAmelCase_ :      List[str]    				=				model_class(__lowercase							).to(__lowercase							)
														lowerCAmelCase_ :      int    				=				model(**__lowercase   ,						output_attentions=__lowercase							)
														self.assertTrue(outputs.attentions is not None							)




						def 						lowercase_ (				self							)					->    List[str]:
										if not self.model_tester.is_training:
														return
										# only MaskFormerForInstanceSegmentation has the loss
										lowerCAmelCase_ :      int    				=				self.all_model_classes[1]
										lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_ :      str    				=				self.model_tester.prepare_config_and_inputs()

										lowerCAmelCase_ :      Optional[Any]    				=				model_class(__lowercase							)
										model.to(__lowercase							)
										model.train()

										lowerCAmelCase_ :      Optional[Any]    				=				model(__lowercase   ,						mask_labels=__lowercase   ,						class_labels=__lowercase							).loss
										loss.backward()




						def 						lowercase_ (				self							)					->    Optional[int]:
										# only MaskFormerForInstanceSegmentation has the loss
										lowerCAmelCase_ :      Any    				=				self.all_model_classes[1]
										lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_				,  lowerCAmelCase_ :      List[str]    				=				self.model_tester.prepare_config_and_inputs()
										lowerCAmelCase_ :      Tuple    				=				True
										lowerCAmelCase_ :      Tuple    				=				True

										lowerCAmelCase_ :      Any    				=				model_class(__lowercase							)
										model.to(__lowercase							)
										model.train()

										lowerCAmelCase_ :      Any    				=				model(__lowercase   ,						mask_labels=__lowercase   ,						class_labels=__lowercase							)

										lowerCAmelCase_ :      Union[str, Any]    				=				outputs.encoder_hidden_states[0]
										encoder_hidden_states.retain_grad()

										lowerCAmelCase_ :      str    				=				outputs.pixel_decoder_hidden_states[0]
										pixel_decoder_hidden_states.retain_grad()
										# we requires_grad=True in inputs_embeds (line 2152), the original implementation don't
										lowerCAmelCase_ :      str    				=				outputs.transformer_decoder_hidden_states[0]
										transformer_decoder_hidden_states.retain_grad()

										lowerCAmelCase_ :      Union[str, Any]    				=				outputs.attentions[0]
										attentions.retain_grad()

										outputs.loss.backward(retain_graph=__lowercase							)

										self.assertIsNotNone(encoder_hidden_states.grad							)
										self.assertIsNotNone(pixel_decoder_hidden_states.grad							)
										self.assertIsNotNone(transformer_decoder_hidden_states.grad							)
										self.assertIsNotNone(attentions.grad							)






_UpperCAmelCase			:  Dict     =1E-4



def 			lowerCAmelCase (		)-> Any:
				lowerCAmelCase_ :      Optional[int]    				=				Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''' )
				return image







@require_vision
@slow
class 				snake_case__(							unittest.TestCase							):







						'''simple docstring'''




						@cached_property
						def 						lowercase_ (				self							)					->    Union[str, Any]:
										return (
										    MaskFormerImageProcessor.from_pretrained('''facebook/maskformer-swin-small-coco'''							)
										    if is_vision_available()
										    else None
										)




						def 						lowercase_ (				self							)					->    Any:
										lowerCAmelCase_ :      Optional[Any]    				=				MaskFormerModel.from_pretrained('''facebook/maskformer-swin-small-coco'''							).to(__lowercase							)
										lowerCAmelCase_ :      Dict    				=				self.default_image_processor
										lowerCAmelCase_ :      int    				=				prepare_img()
										lowerCAmelCase_ :      Any    				=				image_processor(__lowercase   ,						return_tensors='''pt'''							).to(__lowercase							)
										lowerCAmelCase_ :      Any    				=				inputs['''pixel_values'''].shape
										# check size is divisible by 32
										self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0							)
										# check size
										self.assertEqual(__lowercase   ,						(1, 3, 8_0_0, 1_0_8_8)							)

										with torch.no_grad():
														lowerCAmelCase_ :      List[str]    				=				model(**__lowercase							)

										lowerCAmelCase_ :      Union[str, Any]    				=				torch.tensor(
										    [[-0.04_82, 0.92_28, 0.49_51], [-0.25_47, 0.80_17, 0.85_27], [-0.00_69, 0.33_85, -0.00_89]]							).to(__lowercase							)
										self.assertTrue(
										    torch.allclose(
										        outputs.encoder_last_hidden_state[0, 0, :3, :3]   ,						__lowercase   ,						atol=__lowercase							)							)

										lowerCAmelCase_ :      List[Any]    				=				torch.tensor(
										    [[-0.84_22, -0.84_34, -0.97_18], [-1.01_44, -0.55_65, -0.41_95], [-1.00_38, -0.44_84, -0.19_61]]							).to(__lowercase							)
										self.assertTrue(
										    torch.allclose(
										        outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3]   ,						__lowercase   ,						atol=__lowercase							)							)

										lowerCAmelCase_ :      int    				=				torch.tensor(
										    [[0.28_52, -0.01_59, 0.97_35], [0.62_54, 0.18_58, 0.85_29], [-0.06_80, -0.41_16, 1.84_13]]							).to(__lowercase							)
										self.assertTrue(
										    torch.allclose(
										        outputs.transformer_decoder_last_hidden_state[0, :3, :3]   ,						__lowercase   ,						atol=__lowercase							)							)




						def 						lowercase_ (				self							)					->    Dict:
										lowerCAmelCase_ :      Optional[Any]    				=				(
										    MaskFormerForInstanceSegmentation.from_pretrained('''facebook/maskformer-swin-small-coco'''							)
										    .to(__lowercase							)
										    .eval()
										)
										lowerCAmelCase_ :      Tuple    				=				self.default_image_processor
										lowerCAmelCase_ :      Optional[Any]    				=				prepare_img()
										lowerCAmelCase_ :      int    				=				image_processor(__lowercase   ,						return_tensors='''pt'''							).to(__lowercase							)
										lowerCAmelCase_ :      Tuple    				=				inputs['''pixel_values'''].shape
										# check size is divisible by 32
										self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0							)
										# check size
										self.assertEqual(__lowercase   ,						(1, 3, 8_0_0, 1_0_8_8)							)

										with torch.no_grad():
														lowerCAmelCase_ :      Dict    				=				model(**__lowercase							)
										# masks_queries_logits
										lowerCAmelCase_ :      Optional[int]    				=				outputs.masks_queries_logits
										self.assertEqual(
										    masks_queries_logits.shape   ,						(1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)   ,						)
										lowerCAmelCase_ :      Tuple    				=				[
										    [-1.3_73_71_24, -1.7_72_49_37, -1.9_36_42_33],
										    [-1.5_97_72_81, -1.9_86_79_39, -2.1_52_36_95],
										    [-1.5_79_53_98, -1.9_26_98_32, -2.09_39_42],
										]
										lowerCAmelCase_ :      int    				=				torch.tensor(__lowercase							).to(__lowercase							)
										self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]   ,						__lowercase   ,						atol=__lowercase							)							)
										# class_queries_logits
										lowerCAmelCase_ :      List[Any]    				=				outputs.class_queries_logits
										self.assertEqual(
										    class_queries_logits.shape   ,						(1, model.config.decoder_config.num_queries, model.config.num_labels + 1)							)
										lowerCAmelCase_ :      Dict    				=				torch.tensor(
										    [
										        [1.6_512e00, -5.2_572e00, -3.3_519e00],
										        [3.6_169e-02, -5.9_025e00, -2.9_313e00],
										        [1.0_766e-04, -7.7_630e00, -5.1_263e00],
										    ]							).to(__lowercase							)
										self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]   ,						__lowercase   ,						atol=__lowercase							)							)




						def 						lowercase_ (				self							)					->    Optional[Any]:
										lowerCAmelCase_ :      str    				=				(
										    MaskFormerForInstanceSegmentation.from_pretrained('''facebook/maskformer-resnet101-coco-stuff'''							)
										    .to(__lowercase							)
										    .eval()
										)
										lowerCAmelCase_ :      int    				=				self.default_image_processor
										lowerCAmelCase_ :      Optional[Any]    				=				prepare_img()
										lowerCAmelCase_ :      Dict    				=				image_processor(__lowercase   ,						return_tensors='''pt'''							).to(__lowercase							)
										lowerCAmelCase_ :      Optional[Any]    				=				inputs['''pixel_values'''].shape
										# check size is divisible by 32
										self.assertTrue((inputs_shape[-1] % 3_2) == 0 and (inputs_shape[-2] % 3_2) == 0							)
										# check size
										self.assertEqual(__lowercase   ,						(1, 3, 8_0_0, 1_0_8_8)							)

										with torch.no_grad():
														lowerCAmelCase_ :      str    				=				model(**__lowercase							)
										# masks_queries_logits
										lowerCAmelCase_ :      List[str]    				=				outputs.masks_queries_logits
										self.assertEqual(
										    masks_queries_logits.shape   ,						(1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)   ,						)
										lowerCAmelCase_ :      Any    				=				[[-0.90_46, -2.63_66, -4.60_62], [-3.41_79, -5.78_90, -8.80_57], [-4.91_79, -7.65_60, -10.77_11]]
										lowerCAmelCase_ :      str    				=				torch.tensor(__lowercase							).to(__lowercase							)
										self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]   ,						__lowercase   ,						atol=__lowercase							)							)
										# class_queries_logits
										lowerCAmelCase_ :      Optional[int]    				=				outputs.class_queries_logits
										self.assertEqual(
										    class_queries_logits.shape   ,						(1, model.config.decoder_config.num_queries, model.config.num_labels + 1)							)
										lowerCAmelCase_ :      int    				=				torch.tensor(
										    [[4.71_88, -3.25_85, -2.88_57], [6.68_71, -2.91_81, -1.24_87], [7.24_49, -2.27_64, -2.18_74]]							).to(__lowercase							)
										self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]   ,						__lowercase   ,						atol=__lowercase							)							)




						def 						lowercase_ (				self							)					->    Optional[Any]:
										lowerCAmelCase_ :      Dict    				=				(
										    MaskFormerForInstanceSegmentation.from_pretrained('''facebook/maskformer-swin-small-coco'''							)
										    .to(__lowercase							)
										    .eval()
										)
										lowerCAmelCase_ :      str    				=				self.default_image_processor

										lowerCAmelCase_ :      Union[str, Any]    				=				image_processor(
										    [np.zeros((3, 8_0_0, 1_3_3_3)							), np.zeros((3, 8_0_0, 1_3_3_3)							)]   ,						segmentation_maps=[np.zeros((3_8_4, 3_8_4)							).astype(np.floataa							), np.zeros((3_8_4, 3_8_4)							).astype(np.floataa							)]   ,						return_tensors='''pt'''   ,						)

										lowerCAmelCase_ :      Optional[Any]    				=				inputs['''pixel_values'''].to(__lowercase							)
										lowerCAmelCase_ :      int    				=				[el.to(__lowercase							) for el in inputs['''mask_labels''']]
										lowerCAmelCase_ :      Optional[Any]    				=				[el.to(__lowercase							) for el in inputs['''class_labels''']]

										with torch.no_grad():
														lowerCAmelCase_ :      str    				=				model(**__lowercase							)

										self.assertTrue(outputs.loss is not None							)