






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


lowerCamelCase : List[Any]						   =       logging.get_logger(__name__)

lowerCamelCase : str						   =       {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

lowerCamelCase : Optional[int]						   =       {
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

lowerCamelCase : Optional[Any]						   =       {
    "gpt2": 10_24,
    "gpt2-medium": 10_24,
    "gpt2-large": 10_24,
    "gpt2-xl": 10_24,
    "distilgpt2": 10_24,
}

class 		lowerCAmelCase     ( __lowerCAmelCase			):

					'''simple docstring'''

					_A						:			str									=      VOCAB_FILES_NAMES
					_A						:			Optional[Any]									=      PRETRAINED_VOCAB_FILES_MAP
					_A						:			Optional[int]									=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_A						:			Union[str, Any]									=      ['''input_ids''', '''attention_mask''']
					_A						:			int									=      GPTaTokenizer



					def __init__(  self				:	Any		,     __a				:	Union[str, Any]=None		,     __a				:	Union[str, Any]=None		,     __a				:	int=None		,     __a				:	Tuple="<|endoftext|>"		,     __a				:	Optional[Any]="<|endoftext|>"		,     __a				:	List[str]="<|endoftext|>"		,     __a				:	Optional[int]=False		,     **__a				:	Dict		,     )			->  List[str]:


												"""simple docstring"""

												super().__init__(
												    _UpperCamelCase		,     _UpperCamelCase		,     tokenizer_file=_UpperCamelCase		,     unk_token=_UpperCamelCase		,     bos_token=_UpperCamelCase		,     eos_token=_UpperCamelCase		,     add_prefix_space=_UpperCamelCase		,     **_UpperCamelCase		,     )

												__lowercase      :							Optional[Any]				=     kwargs.pop("""add_bos_token"""		,     _UpperCamelCase							)

												__lowercase      :							Optional[Any]				=     json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
												if pre_tok_state.get("""add_prefix_space"""		,     _UpperCamelCase							) != add_prefix_space:
																			__lowercase      :							Optional[int]				=     getattr(_UpperCamelCase		,     pre_tok_state.pop("""type"""							)							)
																			__lowercase      :							Dict				=     add_prefix_space
																			__lowercase      :							List[Any]				=     pre_tok_class(**_UpperCamelCase							)

												__lowercase      :							Optional[Any]				=     add_prefix_space



					def       lowerCAmelCase				(  self				:	List[Any]		,     *__a				:	Optional[int]		,     **__a				:	Optional[int]							)			->  BatchEncoding:


												"""simple docstring"""

												__lowercase      :							int				=     kwargs.get("""is_split_into_words"""		,     _UpperCamelCase							)
												assert self.add_prefix_space or not is_split_into_words, (
												    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
												    "to use it with pretokenized inputs."
												)

												return super()._batch_encode_plus(*_UpperCamelCase		,     **_UpperCamelCase							)



					def       lowerCAmelCase				(  self				:	Dict		,     *__a				:	Dict		,     **__a				:	Dict							)			->  BatchEncoding:


												"""simple docstring"""

												__lowercase      :							List[Any]				=     kwargs.get("""is_split_into_words"""		,     _UpperCamelCase							)

												assert self.add_prefix_space or not is_split_into_words, (
												    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
												    "to use it with pretokenized inputs."
												)

												return super()._encode_plus(*_UpperCamelCase		,     **_UpperCamelCase							)



					def       lowerCAmelCase				(  self				:	Any		,     __a				:	str		,     __a				:	Optional[str] = None							)			->  Tuple[str]:


												"""simple docstring"""

												__lowercase      :							Any				=     self._tokenizer.model.save(_UpperCamelCase		,     name=_UpperCamelCase							)
												return tuple(_UpperCamelCase							)



					def       lowerCAmelCase				(  self				:	Optional[int]		,     __a				:	"Conversation"							)			->  List[int]:


												"""simple docstring"""

												__lowercase      :							List[str]				=     []
												for is_user, text in conversation.iter_texts():
																			input_ids.extend(self.encode(_UpperCamelCase		,     add_special_tokens=_UpperCamelCase							) + [self.eos_token_id]							)

												if len(_UpperCamelCase							) > self.model_max_length:
																			__lowercase      :							str				=     input_ids[-self.model_max_length :]
												return input_ids










import inspect
import unittest

from transformers import MobileNetVaConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import MobileNetVaForImageClassification, MobileNetVaForSemanticSegmentation, MobileNetVaModel
	from transformers.models.mobilenet_va.modeling_mobilenet_va import MOBILENET_V2_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
	from PIL import Image

	from transformers import MobileNetVaImageProcessor

class 		lowerCAmelCase     ( __a			):

					'''simple docstring'''

					def       lowerCAmelCase				(  self				:	List[str]							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							str				=     self.config_class(**self.inputs_dict							)
												self.parent.assertTrue(hasattr(__a		,     """tf_padding"""							)							)
												self.parent.assertTrue(hasattr(__a		,     """depth_multiplier"""							)							)







class 		lowerCAmelCase     :

					'''simple docstring'''

					def __init__(  self				:	Union[str, Any]		,     __a				:	Tuple		,     __a				:	str=13		,     __a				:	Dict=3		,     __a				:	List[Any]=32		,     __a				:	Any=0.25		,     __a				:	Any=8		,     __a				:	Optional[int]=8		,     __a				:	Optional[int]=6		,     __a				:	Dict=32		,     __a				:	Tuple=True		,     __a				:	List[Any]=True		,     __a				:	Optional[int]=True		,     __a				:	Tuple="relu6"		,     __a				:	Optional[Any]=1280		,     __a				:	str=0.1		,     __a				:	str=0.02		,     __a				:	Optional[Any]=True		,     __a				:	Tuple=True		,     __a				:	Dict=10		,     __a				:	Optional[Any]=None		,     )			->  Any:


												"""simple docstring"""

												__lowercase      :							List[str]				=     parent
												__lowercase      :							Tuple				=     batch_size
												__lowercase      :							Dict				=     num_channels
												__lowercase      :							Optional[int]				=     image_size
												__lowercase      :							int				=     depth_multiplier
												__lowercase      :							str				=     depth_divisible_by
												__lowercase      :							int				=     min_depth
												__lowercase      :							Tuple				=     expand_ratio
												__lowercase      :							Optional[int]				=     tf_padding
												__lowercase      :							Dict				=     output_stride
												__lowercase      :							Dict				=     first_layer_is_expansion
												__lowercase      :							Optional[Any]				=     finegrained_output
												__lowercase      :							str				=     hidden_act
												__lowercase      :							Union[str, Any]				=     last_hidden_size if finegrained_output else int(last_hidden_size * depth_multiplier							)
												__lowercase      :							Optional[int]				=     classifier_dropout_prob
												__lowercase      :							int				=     use_labels
												__lowercase      :							Optional[int]				=     is_training
												__lowercase      :							Dict				=     num_labels
												__lowercase      :							Tuple				=     initializer_range
												__lowercase      :							Optional[Any]				=     scope



					def       lowerCAmelCase				(  self				:	Any							)			->  Optional[Any]:


												"""simple docstring"""

												__lowercase      :							Tuple				=     floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

												__lowercase      :							List[Any]				=     None
												__lowercase      :							Optional[Any]				=     None
												if self.use_labels:
																			__lowercase      :							List[Any]				=     ids_tensor([self.batch_size]		,     self.num_labels							)
																			__lowercase      :							Optional[int]				=     ids_tensor([self.batch_size, self.image_size, self.image_size]		,     self.num_labels							)

												__lowercase      :							List[Any]				=     self.get_config()

												return config, pixel_values, labels, pixel_labels



					def       lowerCAmelCase				(  self				:	str							)			->  Union[str, Any]:


												"""simple docstring"""

												return MobileNetVaConfig(
												    num_channels=self.num_channels		,     image_size=self.image_size		,     depth_multiplier=self.depth_multiplier		,     depth_divisible_by=self.depth_divisible_by		,     min_depth=self.min_depth		,     expand_ratio=self.expand_ratio		,     output_stride=self.output_stride		,     first_layer_is_expansion=self.first_layer_is_expansion		,     finegrained_output=self.finegrained_output		,     hidden_act=self.hidden_act		,     tf_padding=self.tf_padding		,     classifier_dropout_prob=self.classifier_dropout_prob		,     initializer_range=self.initializer_range		,     )



					def       lowerCAmelCase				(  self				:	Tuple		,     __a				:	Dict		,     __a				:	Tuple		,     __a				:	Optional[int]		,     __a				:	Union[str, Any]							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							Optional[int]				=     MobileNetVaModel(config=__a							)
												model.to(__a							)
												model.eval()
												__lowercase      :							Tuple				=     model(__a							)
												self.parent.assertEqual(
												    result.last_hidden_state.shape		,     (
												        self.batch_size,
												        self.last_hidden_size,
												        self.image_size // self.output_stride,
												        self.image_size // self.output_stride,
												    )		,     )
												self.parent.assertEqual(
												    result.pooler_output.shape		,     (self.batch_size, self.last_hidden_size)		,     )



					def       lowerCAmelCase				(  self				:	List[str]		,     __a				:	Optional[int]		,     __a				:	List[str]		,     __a				:	str		,     __a				:	Optional[int]							)			->  Tuple:


												"""simple docstring"""

												__lowercase      :							List[Any]				=     self.num_labels
												__lowercase      :							Dict				=     MobileNetVaForImageClassification(__a							)
												model.to(__a							)
												model.eval()
												__lowercase      :							Dict				=     model(__a		,     labels=__a							)
												self.parent.assertEqual(result.logits.shape		,     (self.batch_size, self.num_labels)							)



					def       lowerCAmelCase				(  self				:	int		,     __a				:	List[str]		,     __a				:	Tuple		,     __a				:	Any		,     __a				:	List[str]							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							int				=     self.num_labels
												__lowercase      :							List[Any]				=     MobileNetVaForSemanticSegmentation(__a							)
												model.to(__a							)
												model.eval()
												__lowercase      :							Dict				=     model(__a							)
												self.parent.assertEqual(
												    result.logits.shape		,     (
												        self.batch_size,
												        self.num_labels,
												        self.image_size // self.output_stride,
												        self.image_size // self.output_stride,
												    )		,     )
												__lowercase      :							str				=     model(__a		,     labels=__a							)
												self.parent.assertEqual(
												    result.logits.shape		,     (
												        self.batch_size,
												        self.num_labels,
												        self.image_size // self.output_stride,
												        self.image_size // self.output_stride,
												    )		,     )



					def       lowerCAmelCase				(  self				:	Tuple							)			->  Optional[int]:


												"""simple docstring"""

												__lowercase      :							List[str]				=     self.prepare_config_and_inputs()
												__lowercase			,			__lowercase			,			__lowercase			,			__lowercase      :							List[str]				=     config_and_inputs
												__lowercase      :							List[Any]				=     {"""pixel_values""": pixel_values}
												return config, inputs_dict







@require_torch
class 		lowerCAmelCase     ( __a      ,       __a      ,       unittest.TestCase			):

					'''simple docstring'''

					_A						:			Tuple									=      (
					    (MobileNetVaModel, MobileNetVaForImageClassification, MobileNetVaForSemanticSegmentation)
					    if is_torch_available()
					    else ()
					)
					_A						:			Optional[Any]									=      (
					    {
					        '''feature-extraction''': MobileNetVaModel,
					        '''image-classification''': MobileNetVaForImageClassification,
					        '''image-segmentation''': MobileNetVaForSemanticSegmentation,
					    }
					    if is_torch_available()
					    else {}
					)

					_A						:			Tuple									=      False
					_A						:			List[str]									=      False
					_A						:			List[str]									=      False
					_A						:			Optional[int]									=      False



					def       lowerCAmelCase				(  self				:	Optional[Any]							)			->  List[Any]:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     MobileNetVaModelTester(self							)
												__lowercase      :							int				=     MobileNetVaConfigTester(self		,     config_class=__a		,     has_text_modality=__a							)



					def       lowerCAmelCase				(  self				:	Union[str, Any]							)			->  List[Any]:


												"""simple docstring"""

												self.config_tester.run_common_tests()



					@unittest.skip(reason="""MobileNetV2 does not use inputs_embeds"""							)
					def       lowerCAmelCase				(  self				:	List[str]							)			->  Union[str, Any]:


												"""simple docstring"""

												pass



					@unittest.skip(reason="""MobileNetV2 does not support input and output embeddings"""							)
					def       lowerCAmelCase				(  self				:	Any							)			->  Tuple:


												"""simple docstring"""

												pass



					@unittest.skip(reason="""MobileNetV2 does not output attentions"""							)
					def       lowerCAmelCase				(  self				:	List[str]							)			->  int:


												"""simple docstring"""

												pass



					def       lowerCAmelCase				(  self				:	List[str]							)			->  Dict:


												"""simple docstring"""

												__lowercase			,			__lowercase      :							Optional[int]				=     self.model_tester.prepare_config_and_inputs_for_common()

												for model_class in self.all_model_classes:
																			__lowercase      :							List[Any]				=     model_class(__a							)
																			__lowercase      :							Any				=     inspect.signature(model.forward							)
																			# signature.parameters is an OrderedDict => so arg_names order is deterministic
																			__lowercase      :							int				=     [*signature.parameters.keys()]

																			__lowercase      :							Any				=     ["""pixel_values"""]
																			self.assertListEqual(arg_names[:1]		,     __a							)



					def       lowerCAmelCase				(  self				:	Dict							)			->  Any:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(*__a							)



					def       lowerCAmelCase				(  self				:	List[str]							)			->  Tuple:


												"""simple docstring"""

												def check_hidden_states_output(__a				:	List[Any]		,     __a				:	Tuple		,     __a				:	List[str]							):
																			__lowercase      :							Optional[Any]				=     model_class(__a							)
																			model.to(__a							)
																			model.eval()

																			with torch.no_grad():
																										__lowercase      :							List[Any]				=     model(**self._prepare_for_class(__a		,     __a							)							)

																			__lowercase      :							Tuple				=     outputs.hidden_states

																			__lowercase      :							str				=     16
																			self.assertEqual(len(__a							)		,     __a							)

												__lowercase			,			__lowercase      :							int				=     self.model_tester.prepare_config_and_inputs_for_common()

												for model_class in self.all_model_classes:
																			__lowercase      :							Any				=     True
																			check_hidden_states_output(__a		,     __a		,     __a							)

																			# check that output_hidden_states also work using config
																			del inputs_dict["output_hidden_states"]
																			__lowercase      :							Union[str, Any]				=     True

																			check_hidden_states_output(__a		,     __a		,     __a							)



					def       lowerCAmelCase				(  self				:	Union[str, Any]							)			->  Any:


												"""simple docstring"""

												__lowercase      :							Dict				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_image_classification(*__a							)



					def       lowerCAmelCase				(  self				:	List[str]							)			->  int:


												"""simple docstring"""

												__lowercase      :							Union[str, Any]				=     self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_semantic_segmentation(*__a							)



					@slow
					def       lowerCAmelCase				(  self				:	Optional[int]							)			->  Optional[Any]:


												"""simple docstring"""

												for model_name in MOBILENET_V2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																			__lowercase      :							Optional[int]				=     MobileNetVaModel.from_pretrained(__a							)
																			self.assertIsNotNone(__a							)







def       snake_case_		(     ):
							__lowercase      :							List[Any]				=     Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""      )
							return image



@require_torch
@require_vision
class 		lowerCAmelCase     ( unittest.TestCase			):

					'''simple docstring'''

					@cached_property
					def       lowerCAmelCase				(  self				:	Any							)			->  Union[str, Any]:


												"""simple docstring"""

												return (
												    MobileNetVaImageProcessor.from_pretrained("""google/mobilenet_v2_1.0_224"""							) if is_vision_available() else None
												)



					@slow
					def       lowerCAmelCase				(  self				:	str							)			->  int:


												"""simple docstring"""

												__lowercase      :							Tuple				=     MobileNetVaForImageClassification.from_pretrained("""google/mobilenet_v2_1.0_224"""							).to(__a							)

												__lowercase      :							str				=     self.default_image_processor
												__lowercase      :							Tuple				=     prepare_img()
												__lowercase      :							Tuple				=     image_processor(images=__a		,     return_tensors="""pt"""							).to(__a							)

												# forward pass
												with torch.no_grad():
																			__lowercase      :							str				=     model(**__a							)

												# verify the logits
												__lowercase      :							Union[str, Any]				=     torch.Size((1, 1001)							)
												self.assertEqual(outputs.logits.shape		,     __a							)

												__lowercase      :							str				=     torch.tensor([0.2445, -1.1993, 0.1905]							).to(__a							)

												self.assertTrue(torch.allclose(outputs.logits[0, :3]		,     __a		,     atol=1E-4							)							)



					@slow
					def       lowerCAmelCase				(  self				:	Tuple							)			->  Any:


												"""simple docstring"""

												__lowercase      :							int				=     MobileNetVaForSemanticSegmentation.from_pretrained("""google/deeplabv3_mobilenet_v2_1.0_513"""							)
												__lowercase      :							Dict				=     model.to(__a							)

												__lowercase      :							Tuple				=     MobileNetVaImageProcessor.from_pretrained("""google/deeplabv3_mobilenet_v2_1.0_513"""							)

												__lowercase      :							List[str]				=     prepare_img()
												__lowercase      :							Optional[int]				=     image_processor(images=__a		,     return_tensors="""pt"""							).to(__a							)

												# forward pass
												with torch.no_grad():
																			__lowercase      :							Union[str, Any]				=     model(**__a							)
												__lowercase      :							Any				=     outputs.logits

												# verify the logits
												__lowercase      :							Dict				=     torch.Size((1, 21, 65, 65)							)
												self.assertEqual(logits.shape		,     __a							)

												__lowercase      :							str				=     torch.tensor(
												    [
												        [[17.5790, 17.7581, 18.3355], [18.3257, 18.4230, 18.8973], [18.6169, 18.8650, 19.2187]],
												        [[-2.1595, -2.0977, -2.3741], [-2.4226, -2.3028, -2.6835], [-2.7819, -2.5991, -2.7706]],
												        [[4.2058, 4.8317, 4.7638], [4.4136, 5.0361, 4.9383], [4.5028, 4.9644, 4.8734]],
												    ]		,     device=__a		,     )

												self.assertTrue(torch.allclose(logits[0, :3, :3, :3]		,     __a		,     atol=1E-4							)							)