


'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_pegasus import PegasusTokenizer
else:
       lowerCAmelCase    :     Optional[int]					=						None


lowerCAmelCase    :     List[Any]					=						logging.get_logger(__name__)


lowerCAmelCase    :     str					=						"""▁"""

lowerCAmelCase    :     str					=						{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase    :     List[str]					=						{
    """vocab_file""": {"""google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"""},
    """tokenizer_file""": {
        """google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"""
    },
}

lowerCAmelCase    :     Tuple					=						{
    """google/pegasus-xsum""": 5_12,
}
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     __magic_name__    			=    VOCAB_FILES_NAMES
     __magic_name__    			=    PRETRAINED_VOCAB_FILES_MAP
     __magic_name__    			=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__    			=    PegasusTokenizer
     __magic_name__    			=    ["input_ids", "attention_mask"]







     def __init__( self      ,	snake_case__=None      ,	snake_case__=None      ,	snake_case__="<pad>"      ,	snake_case__="</s>"      ,	snake_case__="<unk>"      ,	snake_case__="<mask_2>"      ,	snake_case__="<mask_1>"      ,	snake_case__=None      ,	snake_case__=103      ,	**snake_case__      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       offset

            if additional_special_tokens is not None:
                   if not isinstance(snake_case__      ,	snake_case__   ):
                          raise TypeError(
                              F'additional_special_tokens should be of type {type(snake_case__   )}, but is'
                              F' {type(snake_case__   )}'   )

                   _lowerCAmelCase    :							Tuple   			=       (
                       ([mask_token_sent] + additional_special_tokens)
                       if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                       else additional_special_tokens
                   )
                   # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
                   additional_special_tokens_extended += [
                       F'<unk_{i}>' for i in range(len(snake_case__   )      ,	self.offset - 1   )
                   ]

                   if len(set(snake_case__   )   ) != len(snake_case__   ):
                          raise ValueError(
                              'Please make sure that the provided additional_special_tokens do not contain an incorrectly'
                              F' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'   )
                   _lowerCAmelCase    :							Any   			=       additional_special_tokens_extended
            else:
                   _lowerCAmelCase    :							int   			=       [mask_token_sent] if mask_token_sent is not None else []
                   additional_special_tokens += [F'<unk_{i}>' for i in range(2      ,	self.offset   )]

            super().__init__(
                snake_case__      ,	tokenizer_file=snake_case__      ,	pad_token=snake_case__      ,	eos_token=snake_case__      ,	unk_token=snake_case__      ,	mask_token=snake_case__      ,	mask_token_sent=snake_case__      ,	offset=snake_case__      ,	additional_special_tokens=snake_case__      ,	**snake_case__      ,	)
            _lowerCAmelCase    :							Union[str, Any]   			=       vocab_file
            _lowerCAmelCase    :							str   			=       False if not self.vocab_file else True







     def a	( self      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Union[str, Any]   			=       set(self.all_special_ids   )  # call it once instead of inside list comp
            all_special_ids.remove(self.unk_token_id   )  # <unk> is only sometimes special

            if all_special_ids != set(range(len(self.additional_special_tokens   ) + 3   )   ):
                   raise ValueError(
                       'There should be 3 special tokens: mask_token, pad_token, and eos_token +'
                       F' {len(self.additional_special_tokens   )} additional_special_tokens, but got {all_special_ids}'   )

            return [1 if x in all_special_ids else 0 for x in seq]







     def a	( self      ,	snake_case__      ,	snake_case__ = None      ,	snake_case__ = False   ):

            '''simple docstring'''

            if already_has_special_tokens:
                   return self._special_token_mask(snake_case__   )
            elif token_ids_a is None:
                   return self._special_token_mask(snake_case__   ) + [1]
            else:
                   return self._special_token_mask(token_ids_a + token_ids_a   ) + [1]







     def a	( self      ,	snake_case__      ,	snake_case__=None   ):

            '''simple docstring'''

            if token_ids_a is None:
                   return token_ids_a + [self.eos_token_id]
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return token_ids_a + token_ids_a + [self.eos_token_id]







     def a	( self      ,	snake_case__      ,	snake_case__ = None   ):

            '''simple docstring'''

            if not self.can_save_slow_tokenizer:
                   raise ValueError(
                       'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
                       'tokenizer.'   )

            if not os.path.isdir(snake_case__   ):
                   logger.error(F'Vocabulary path ({save_directory}) should be a directory'   )
                   return
            _lowerCAmelCase    :							Union[str, Any]   			=       os.path.join(
                snake_case__      ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

            if os.path.abspath(self.vocab_file   ) != os.path.abspath(snake_case__   ):
                   copyfile(self.vocab_file      ,	snake_case__   )

            return (out_vocab_file,)





'''simple docstring'''







import inspect
import unittest

from transformers import MobileViTVaConfig
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import MobileViTVaForImageClassification, MobileViTVaForSemanticSegmentation, MobileViTVaModel
       from transformers.models.mobilevitva.modeling_mobilevitva import (
           MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST,
           make_divisible,
       )


if is_vision_available():
       from PIL import Image

       from transformers import MobileViTImageProcessor
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_	):



     """simple docstring"""






     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       self.config_class(**self.inputs_dict   )
            self.parent.assertTrue(hasattr(snake_case__      ,	'width_multiplier'   )   )
class 	UpperCamelCase__					:



     """simple docstring"""






     def __init__( self      ,	snake_case__      ,	snake_case__=13      ,	snake_case__=64      ,	snake_case__=2      ,	snake_case__=3      ,	snake_case__="swish"      ,	snake_case__=3      ,	snake_case__=32      ,	snake_case__=0.1      ,	snake_case__=0.02      ,	snake_case__=True      ,	snake_case__=True      ,	snake_case__=10      ,	snake_case__=None      ,	snake_case__=0.25      ,	snake_case__=0.0      ,	snake_case__=0.0      ,	):

            '''simple docstring'''

            _lowerCAmelCase    :							Tuple   			=       parent
            _lowerCAmelCase    :							Optional[int]   			=       batch_size
            _lowerCAmelCase    :							List[Any]   			=       image_size
            _lowerCAmelCase    :							List[Any]   			=       patch_size
            _lowerCAmelCase    :							List[str]   			=       num_channels
            _lowerCAmelCase    :							Optional[Any]   			=       make_divisible(512 * width_multiplier      ,	divisor=8   )
            _lowerCAmelCase    :							Optional[Any]   			=       hidden_act
            _lowerCAmelCase    :							List[Any]   			=       conv_kernel_size
            _lowerCAmelCase    :							Optional[Any]   			=       output_stride
            _lowerCAmelCase    :							List[Any]   			=       classifier_dropout_prob
            _lowerCAmelCase    :							str   			=       use_labels
            _lowerCAmelCase    :							List[str]   			=       is_training
            _lowerCAmelCase    :							Optional[int]   			=       num_labels
            _lowerCAmelCase    :							List[str]   			=       initializer_range
            _lowerCAmelCase    :							str   			=       scope
            _lowerCAmelCase    :							Any   			=       width_multiplier
            _lowerCAmelCase    :							Union[str, Any]   			=       ffn_dropout
            _lowerCAmelCase    :							Optional[int]   			=       attn_dropout







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[Any]   			=       floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]   )

            _lowerCAmelCase    :							Optional[Any]   			=       None
            _lowerCAmelCase    :							Dict   			=       None
            if self.use_labels:
                   _lowerCAmelCase    :							Optional[Any]   			=       ids_tensor([self.batch_size]      ,	self.num_labels   )
                   _lowerCAmelCase    :							Optional[Any]   			=       ids_tensor([self.batch_size, self.image_size, self.image_size]      ,	self.num_labels   )

            _lowerCAmelCase    :							Optional[int]   			=       self.get_config()

            return config, pixel_values, labels, pixel_labels







     def a	( self   ):

            '''simple docstring'''

            return MobileViTVaConfig(
                image_size=self.image_size      ,	patch_size=self.patch_size      ,	num_channels=self.num_channels      ,	hidden_act=self.hidden_act      ,	conv_kernel_size=self.conv_kernel_size      ,	output_stride=self.output_stride      ,	classifier_dropout_prob=self.classifier_dropout_prob      ,	initializer_range=self.initializer_range      ,	width_multiplier=self.width_multiplier      ,	ffn_dropout=self.ffn_dropout_prob      ,	attn_dropout=self.attn_dropout_prob      ,	)







     def a	( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       MobileViTVaModel(config=snake_case__   )
            model.to(snake_case__   )
            model.eval()
            _lowerCAmelCase    :							str   			=       model(snake_case__   )
            self.parent.assertEqual(
                result.last_hidden_state.shape      ,	(
                    self.batch_size,
                    self.last_hidden_size,
                    self.image_size // self.output_stride,
                    self.image_size // self.output_stride,
                )      ,	)







     def a	( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       self.num_labels
            _lowerCAmelCase    :							List[Any]   			=       MobileViTVaForImageClassification(snake_case__   )
            model.to(snake_case__   )
            model.eval()
            _lowerCAmelCase    :							int   			=       model(snake_case__      ,	labels=snake_case__   )
            self.parent.assertEqual(result.logits.shape      ,	(self.batch_size, self.num_labels)   )







     def a	( self      ,	snake_case__      ,	snake_case__      ,	snake_case__      ,	snake_case__   ):

            '''simple docstring'''

            _lowerCAmelCase    :							List[Any]   			=       self.num_labels
            _lowerCAmelCase    :							Optional[int]   			=       MobileViTVaForSemanticSegmentation(snake_case__   )
            model.to(snake_case__   )
            model.eval()
            _lowerCAmelCase    :							Dict   			=       model(snake_case__   )
            self.parent.assertEqual(
                result.logits.shape      ,	(
                    self.batch_size,
                    self.num_labels,
                    self.image_size // self.output_stride,
                    self.image_size // self.output_stride,
                )      ,	)
            _lowerCAmelCase    :							Any   			=       model(snake_case__      ,	labels=snake_case__   )
            self.parent.assertEqual(
                result.logits.shape      ,	(
                    self.batch_size,
                    self.num_labels,
                    self.image_size // self.output_stride,
                    self.image_size // self.output_stride,
                )      ,	)







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       self.prepare_config_and_inputs()
            _lowerCAmelCase			,     _lowerCAmelCase			,     _lowerCAmelCase			,     _lowerCAmelCase    :							Optional[int]   			=       config_and_inputs
            _lowerCAmelCase    :							Tuple   			=       {'pixel_values': pixel_values}
            return config, inputs_dict
@require_torch
class 	UpperCamelCase__					( SCREAMING_SNAKE_CASE_    ,					SCREAMING_SNAKE_CASE_    ,					unittest.TestCase	):



     """simple docstring"""






     __magic_name__    			=    (
         (MobileViTVaModel, MobileViTVaForImageClassification, MobileViTVaForSemanticSegmentation)
         if is_torch_available()
         else ()
     )

     __magic_name__    			=    (
         {
             "feature-extraction": MobileViTVaModel,
             "image-classification": MobileViTVaForImageClassification,
             "image-segmentation": MobileViTVaForSemanticSegmentation,
         }
         if is_torch_available()
         else {}
     )

     __magic_name__    			=    False
     __magic_name__    			=    False
     __magic_name__    			=    False
     __magic_name__    			=    False







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							int   			=       MobileViTVaModelTester(self   )
            _lowerCAmelCase    :							Dict   			=       MobileViTVaConfigTester(self      ,	config_class=snake_case__      ,	has_text_modality=snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            self.config_tester.run_common_tests()







     @unittest.skip(reason='MobileViTV2 does not use inputs_embeds'   )
     def a	( self   ):

            '''simple docstring'''

            pass







     @unittest.skip(reason='MobileViTV2 does not support input and output embeddings'   )
     def a	( self   ):

            '''simple docstring'''

            pass







     @unittest.skip(reason='MobileViTV2 does not output attentions'   )
     def a	( self   ):

            '''simple docstring'''

            pass







     @require_torch_multi_gpu
     @unittest.skip(reason='Got `CUDA error: misaligned address` for tests after this one being run.'   )
     def a	( self   ):

            '''simple docstring'''

            pass







     @unittest.skip('Will be fixed soon by reducing the size of the model used for common tests.'   )
     def a	( self   ):

            '''simple docstring'''

            pass







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase			,     _lowerCAmelCase    :							Optional[int]   			=       self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                   _lowerCAmelCase    :							str   			=       model_class(snake_case__   )
                   _lowerCAmelCase    :							List[str]   			=       inspect.signature(model.forward   )
                   # signature.parameters is an OrderedDict => so arg_names order is deterministic
                   _lowerCAmelCase    :							int   			=       [*signature.parameters.keys()]

                   _lowerCAmelCase    :							Any   			=       ['pixel_values']
                   self.assertListEqual(arg_names[:1]      ,	snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Optional[int]   			=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            def check_hidden_states_output(snake_case__      ,	snake_case__      ,	snake_case__   ):
                   _lowerCAmelCase    :							Dict   			=       model_class(snake_case__   )
                   model.to(snake_case__   )
                   model.eval()

                   with torch.no_grad():
                          _lowerCAmelCase    :							Optional[Any]   			=       model(**self._prepare_for_class(snake_case__      ,	snake_case__   )   )

                   _lowerCAmelCase    :							List[str]   			=       outputs.hidden_states

                   _lowerCAmelCase    :							List[str]   			=       5
                   self.assertEqual(len(snake_case__   )      ,	snake_case__   )

                   # MobileViTV2's feature maps are of shape (batch_size, num_channels, height, width)
                   # with the width and height being successively divided by 2.
                   _lowerCAmelCase    :							List[Any]   			=       2
                   for i in range(len(snake_case__   )   ):
                          self.assertListEqual(
                              list(hidden_states[i].shape[-2:]   )      ,	[self.model_tester.image_size // divisor, self.model_tester.image_size // divisor]      ,	)
                          divisor *= 2

                   self.assertEqual(self.model_tester.output_stride      ,	divisor // 2   )

            _lowerCAmelCase			,     _lowerCAmelCase    :							Optional[int]   			=       self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                   _lowerCAmelCase    :							Optional[int]   			=       True
                   check_hidden_states_output(snake_case__      ,	snake_case__      ,	snake_case__   )

                   # check that output_hidden_states also work using config
                   del inputs_dict["output_hidden_states"]
                   _lowerCAmelCase    :							Any   			=       True

                   check_hidden_states_output(snake_case__      ,	snake_case__      ,	snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Dict   			=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_image_classification(*snake_case__   )







     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_semantic_segmentation(*snake_case__   )







     @slow
     def a	( self   ):

            '''simple docstring'''

            for model_name in MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                   _lowerCAmelCase    :							Dict   			=       MobileViTVaModel.from_pretrained(snake_case__   )
                   self.assertIsNotNone(snake_case__   )






def 			lowercase				():







       """simple docstring"""







       _lowerCAmelCase    :							Any   			=       Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'					)
       return image
@require_torch
@require_vision
class 	UpperCamelCase__					( unittest.TestCase	):



     """simple docstring"""






     @cached_property
     def a	( self   ):

            '''simple docstring'''

            return (
                MobileViTImageProcessor.from_pretrained('apple/mobilevitv2-1.0-imagenet1k-256'   )
                if is_vision_available()
                else None
            )







     @slow
     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Union[str, Any]   			=       MobileViTVaForImageClassification.from_pretrained('apple/mobilevitv2-1.0-imagenet1k-256'   ).to(
                snake_case__   )

            _lowerCAmelCase    :							str   			=       self.default_image_processor
            _lowerCAmelCase    :							Any   			=       prepare_img()
            _lowerCAmelCase    :							Optional[int]   			=       image_processor(images=snake_case__      ,	return_tensors='pt'   ).to(snake_case__   )

            # forward pass
            with torch.no_grad():
                   _lowerCAmelCase    :							Tuple   			=       model(**snake_case__   )

            # verify the logits
            _lowerCAmelCase    :							Optional[Any]   			=       torch.Size((1, 1000)   )
            self.assertEqual(outputs.logits.shape      ,	snake_case__   )

            _lowerCAmelCase    :							Tuple   			=       torch.tensor([-1.6336E00, -7.3204E-02, -5.1883E-01]   ).to(snake_case__   )

            self.assertTrue(torch.allclose(outputs.logits[0, :3]      ,	snake_case__      ,	atol=1E-4   )   )







     @slow
     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							List[str]   			=       MobileViTVaForSemanticSegmentation.from_pretrained('shehan97/mobilevitv2-1.0-voc-deeplabv3'   )
            _lowerCAmelCase    :							Any   			=       model.to(snake_case__   )

            _lowerCAmelCase    :							int   			=       MobileViTImageProcessor.from_pretrained('shehan97/mobilevitv2-1.0-voc-deeplabv3'   )

            _lowerCAmelCase    :							Optional[int]   			=       prepare_img()
            _lowerCAmelCase    :							Dict   			=       image_processor(images=snake_case__      ,	return_tensors='pt'   ).to(snake_case__   )

            # forward pass
            with torch.no_grad():
                   _lowerCAmelCase    :							int   			=       model(**snake_case__   )
            _lowerCAmelCase    :							Dict   			=       outputs.logits

            # verify the logits
            _lowerCAmelCase    :							str   			=       torch.Size((1, 21, 32, 32)   )
            self.assertEqual(logits.shape      ,	snake_case__   )

            _lowerCAmelCase    :							Any   			=       torch.tensor(
                [
                    [[7.0863, 7.1525, 6.8201], [6.6931, 6.8770, 6.8933], [6.2978, 7.0366, 6.9636]],
                    [[-3.7134, -3.6712, -3.6675], [-3.5825, -3.3549, -3.4777], [-3.3435, -3.3979, -3.2857]],
                    [[-2.9329, -2.8003, -2.7369], [-3.0564, -2.4780, -2.0207], [-2.6889, -1.9298, -1.7640]],
                ]      ,	device=snake_case__      ,	)

            self.assertTrue(torch.allclose(logits[0, :3, :3, :3]      ,	snake_case__      ,	atol=1E-4   )   )







     @slow
     def a	( self   ):

            '''simple docstring'''

            _lowerCAmelCase    :							Any   			=       MobileViTVaForSemanticSegmentation.from_pretrained('shehan97/mobilevitv2-1.0-voc-deeplabv3'   )
            _lowerCAmelCase    :							List[Any]   			=       model.to(snake_case__   )

            _lowerCAmelCase    :							str   			=       MobileViTImageProcessor.from_pretrained('shehan97/mobilevitv2-1.0-voc-deeplabv3'   )

            _lowerCAmelCase    :							Tuple   			=       prepare_img()
            _lowerCAmelCase    :							List[str]   			=       image_processor(images=snake_case__      ,	return_tensors='pt'   ).to(snake_case__   )

            # forward pass
            with torch.no_grad():
                   _lowerCAmelCase    :							Any   			=       model(**snake_case__   )

            _lowerCAmelCase    :							Optional[Any]   			=       outputs.logits.detach().cpu()

            _lowerCAmelCase    :							Any   			=       image_processor.post_process_semantic_segmentation(outputs=snake_case__      ,	target_sizes=[(50, 60)]   )
            _lowerCAmelCase    :							List[Any]   			=       torch.Size((50, 60)   )
            self.assertEqual(segmentation[0].shape      ,	snake_case__   )

            _lowerCAmelCase    :							List[str]   			=       image_processor.post_process_semantic_segmentation(outputs=snake_case__   )
            _lowerCAmelCase    :							Tuple   			=       torch.Size((32, 32)   )
            self.assertEqual(segmentation[0].shape      ,	snake_case__   )

