


"""simple docstring"""


import collections
import inspect
import unittest

from transformers import FocalNetConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch
		from torch import nn

		from transformers import (
		    FocalNetBackbone,
		    FocalNetForImageClassification,
		    FocalNetForMaskedImageModeling,
		    FocalNetModel,
		)
		from transformers.models.focalnet.modeling_focalnet import FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
		from PIL import Image

		from transformers import AutoImageProcessor
class snake_case      :

		'''simple docstring'''





		def __init__(					self      :       int    ,   a__      :       Optional[int]    ,   a__      :       Any=13    ,   a__      :       str=32    ,   a__      :       Tuple=2    ,   a__      :       Optional[Any]=3    ,   a__      :       Optional[int]=16    ,   a__      :       Union[str, Any]=[32, 64, 1_28]    ,   a__      :       str=[1, 2, 1]    ,   a__      :       Union[str, Any]=[2, 2, 4]    ,   a__      :       List[Any]=2    ,   a__      :       Dict=2.0    ,   a__      :       Optional[Any]=True    ,   a__      :       Optional[int]=0.0    ,   a__      :       Tuple=0.0    ,   a__      :       Tuple=0.1    ,   a__      :       Any="gelu"    ,   a__      :       str=False    ,   a__      :       str=True    ,   a__      :       Dict=0.0_2    ,   a__      :       Optional[int]=1E-5    ,   a__      :       int=True    ,   a__      :       List[str]=None    ,   a__      :       Any=True    ,   a__      :       Optional[Any]=10    ,   a__      :       List[str]=8    ,   a__      :       Tuple=["stage1", "stage2"]    ,   a__      :       List[str]=[1, 2]    ,   )       ->						str:





						'''simple docstring'''







						_A					  =			parent
						_A					  =			batch_size
						_A					  =			image_size
						_A					  =			patch_size
						_A					  =			num_channels
						_A					  =			embed_dim
						_A					  =			hidden_sizes
						_A					  =			depths
						_A					  =			num_heads
						_A					  =			window_size
						_A					  =			mlp_ratio
						_A					  =			qkv_bias
						_A					  =			hidden_dropout_prob
						_A					  =			attention_probs_dropout_prob
						_A					  =			drop_path_rate
						_A					  =			hidden_act
						_A					  =			use_absolute_embeddings
						_A					  =			patch_norm
						_A					  =			layer_norm_eps
						_A					  =			initializer_range
						_A					  =			is_training
						_A					  =			scope
						_A					  =			use_labels
						_A					  =			type_sequence_label_size
						_A					  =			encoder_stride
						_A					  =			out_features
						_A					  =			out_indices







		def  a_ (					self      :       Optional[Any]		)       ->						Union[str, Any]:





						'''simple docstring'''







						_A					  =			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]		)

						_A					  =			None
						if self.use_labels:
										_A					  =			ids_tensor([self.batch_size]    ,   self.type_sequence_label_size		)

						_A					  =			self.get_config()

						return config, pixel_values, labels







		def  a_ (					self      :       str		)       ->						Optional[Any]:





						'''simple docstring'''







						return FocalNetConfig(
						    image_size=self.image_size    ,   patch_size=self.patch_size    ,   num_channels=self.num_channels    ,   embed_dim=self.embed_dim    ,   hidden_sizes=self.hidden_sizes    ,   depths=self.depths    ,   num_heads=self.num_heads    ,   window_size=self.window_size    ,   mlp_ratio=self.mlp_ratio    ,   qkv_bias=self.qkv_bias    ,   hidden_dropout_prob=self.hidden_dropout_prob    ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,   drop_path_rate=self.drop_path_rate    ,   hidden_act=self.hidden_act    ,   use_absolute_embeddings=self.use_absolute_embeddings    ,   path_norm=self.patch_norm    ,   layer_norm_eps=self.layer_norm_eps    ,   initializer_range=self.initializer_range    ,   encoder_stride=self.encoder_stride    ,   out_features=self.out_features    ,   out_indices=self.out_indices    ,   )







		def  a_ (					self      :       Union[str, Any]    ,   a__      :       Any    ,   a__      :       Tuple    ,   a__      :       Optional[Any]		)       ->						Optional[Any]:





						'''simple docstring'''







						_A					  =			FocalNetModel(config=__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()
						_A					  =			model(__UpperCamelCase		)

						_A					  =			((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths		) - 1))
						_A					  =			int(config.embed_dim * 2 ** (len(config.depths		) - 1)		)

						self.parent.assertEqual(result.last_hidden_state.shape    ,   (self.batch_size, expected_seq_len, expected_dim)		)







		def  a_ (					self      :       List[Any]    ,   a__      :       List[str]    ,   a__      :       Union[str, Any]    ,   a__      :       Tuple		)       ->						str:





						'''simple docstring'''







						_A					  =			FocalNetBackbone(config=__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()
						_A					  =			model(__UpperCamelCase		)

						# verify feature maps
						self.parent.assertEqual(len(result.feature_maps		)    ,   len(config.out_features		)		)
						self.parent.assertListEqual(list(result.feature_maps[0].shape		)    ,   [self.batch_size, self.image_size, 8, 8]		)

						# verify channels
						self.parent.assertEqual(len(model.channels		)    ,   len(config.out_features		)		)
						self.parent.assertListEqual(model.channels    ,   config.hidden_sizes[:-1]		)

						# verify backbone works with out_features=None
						_A					  =			None
						_A					  =			FocalNetBackbone(config=__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()
						_A					  =			model(__UpperCamelCase		)

						# verify feature maps
						self.parent.assertEqual(len(result.feature_maps		)    ,   1		)
						self.parent.assertListEqual(list(result.feature_maps[0].shape		)    ,   [self.batch_size, self.image_size * 2, 4, 4]		)

						# verify channels
						self.parent.assertEqual(len(model.channels		)    ,   1		)
						self.parent.assertListEqual(model.channels    ,   [config.hidden_sizes[-1]]		)







		def  a_ (					self      :       Optional[Any]    ,   a__      :       str    ,   a__      :       List[Any]    ,   a__      :       List[str]		)       ->						int:





						'''simple docstring'''







						_A					  =			FocalNetForMaskedImageModeling(config=__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()
						_A					  =			model(__UpperCamelCase		)
						self.parent.assertEqual(
						    result.reconstruction.shape    ,   (self.batch_size, self.num_channels, self.image_size, self.image_size)		)

						# test greyscale images
						_A					  =			1
						_A					  =			FocalNetForMaskedImageModeling(__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()

						_A					  =			floats_tensor([self.batch_size, 1, self.image_size, self.image_size]		)
						_A					  =			model(__UpperCamelCase		)
						self.parent.assertEqual(result.reconstruction.shape    ,   (self.batch_size, 1, self.image_size, self.image_size)		)







		def  a_ (					self      :       str    ,   a__      :       Union[str, Any]    ,   a__      :       List[Any]    ,   a__      :       Optional[int]		)       ->						List[str]:





						'''simple docstring'''







						_A					  =			self.type_sequence_label_size
						_A					  =			FocalNetForImageClassification(__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()
						_A					  =			model(__UpperCamelCase    ,   labels=__UpperCamelCase		)
						self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.type_sequence_label_size)		)

						# test greyscale images
						_A					  =			1
						_A					  =			FocalNetForImageClassification(__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()

						_A					  =			floats_tensor([self.batch_size, 1, self.image_size, self.image_size]		)
						_A					  =			model(__UpperCamelCase		)
						self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.type_sequence_label_size)		)







		def  a_ (					self      :       Optional[int]		)       ->						Tuple:





						'''simple docstring'''







						_A					  =			self.prepare_config_and_inputs()

						_A  ,		_A  ,		_A					  =			config_and_inputs
						_A					  =			{"pixel_values": pixel_values}
						return config, inputs_dict





@require_torch
class snake_case      (  _UpperCAmelCase     ,				_UpperCAmelCase     ,				unittest.TestCase):

		'''simple docstring'''





		__UpperCamelCase           =			(
		    (
		        FocalNetModel,
		        FocalNetForImageClassification,
		        FocalNetForMaskedImageModeling,
		        FocalNetBackbone,
		    )
		    if is_torch_available()
		    else ()
		)
		__UpperCamelCase           =			(
		    {'feature-extraction': FocalNetModel, 'image-classification': FocalNetForImageClassification}
		    if is_torch_available()
		    else {}
		)
		__UpperCamelCase           =			False

		__UpperCamelCase           =			False
		__UpperCamelCase           =			False
		__UpperCamelCase           =			False
		__UpperCamelCase           =			False







		def  a_ (					self      :       Union[str, Any]		)       ->						Optional[int]:





						'''simple docstring'''







						_A					  =			FocalNetModelTester(self		)
						_A					  =			ConfigTester(self    ,   config_class=__UpperCamelCase    ,   embed_dim=37    ,   has_text_modality=__UpperCamelCase		)







		def  a_ (					self      :       Optional[int]		)       ->						Optional[int]:





						'''simple docstring'''







						self.create_and_test_config_common_properties()
						self.config_tester.create_and_test_config_to_json_string()
						self.config_tester.create_and_test_config_to_json_file()
						self.config_tester.create_and_test_config_from_and_save_pretrained()
						self.config_tester.create_and_test_config_with_num_labels()
						self.config_tester.check_config_can_be_init_without_params()
						self.config_tester.check_config_arguments_init()







		def  a_ (					self      :       Dict		)       ->						List[Any]:





						'''simple docstring'''







						return







		def  a_ (					self      :       Union[str, Any]		)       ->						Optional[int]:





						'''simple docstring'''







						_A					  =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*__UpperCamelCase		)







		def  a_ (					self      :       str		)       ->						Dict:





						'''simple docstring'''







						_A					  =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_backbone(*__UpperCamelCase		)







		def  a_ (					self      :       Union[str, Any]		)       ->						Optional[Any]:





						'''simple docstring'''







						_A					  =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_image_modeling(*__UpperCamelCase		)







		def  a_ (					self      :       Tuple		)       ->						Any:





						'''simple docstring'''







						_A					  =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_image_classification(*__UpperCamelCase		)







		@unittest.skip(reason="FocalNet does not use inputs_embeds"		)
		def  a_ (					self      :       List[Any]		)       ->						Any:





						'''simple docstring'''







						pass







		@unittest.skip(reason="FocalNet does not use feedforward chunking"		)
		def  a_ (					self      :       Tuple		)       ->						List[str]:





						'''simple docstring'''







						pass







		def  a_ (					self      :       List[Any]		)       ->						Union[str, Any]:





						'''simple docstring'''







						_A  ,		_A					  =			self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes[:-1]:
										_A					  =			model_class(__UpperCamelCase		)
										self.assertIsInstance(model.get_input_embeddings()    ,   (nn.Module)		)
										_A					  =			model.get_output_embeddings()
										self.assertTrue(x is None or isinstance(__UpperCamelCase    ,   nn.Linear		)		)







		def  a_ (					self      :       Optional[Any]		)       ->						Dict:





						'''simple docstring'''







						_A  ,		_A					  =			self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes[:-1]:
										_A					  =			model_class(__UpperCamelCase		)
										_A					  =			inspect.signature(model.forward		)
										# signature.parameters is an OrderedDict => so arg_names order is deterministic
										_A					  =			[*signature.parameters.keys()]

										_A					  =			["pixel_values"]
										self.assertListEqual(arg_names[:1]    ,   __UpperCamelCase		)







		def  a_ (					self      :       Any    ,   a__      :       Union[str, Any]    ,   a__      :       List[str]    ,   a__      :       List[str]    ,   a__      :       List[Any]		)       ->						List[str]:





						'''simple docstring'''







						_A					  =			model_class(__UpperCamelCase		)
						model.to(__UpperCamelCase		)
						model.eval()

						with torch.no_grad():
										_A					  =			model(**self._prepare_for_class(__UpperCamelCase    ,   __UpperCamelCase		)		)

						_A					  =			outputs.hidden_states

						_A					  =			getattr(
						    self.model_tester    ,   "expected_num_hidden_layers"    ,   len(self.model_tester.depths		) + 1		)
						self.assertEqual(len(__UpperCamelCase		)    ,   __UpperCamelCase		)

						# FocalNet has a different seq_length
						_A					  =			(
						    config.patch_size
						    if isinstance(config.patch_size    ,   collections.abc.Iterable		)
						    else (config.patch_size, config.patch_size)
						)

						_A					  =			(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

						self.assertListEqual(
						    list(hidden_states[0].shape[-2:]		)    ,   [num_patches, self.model_tester.embed_dim]    ,   )

						_A					  =			outputs.reshaped_hidden_states
						self.assertEqual(len(__UpperCamelCase		)    ,   __UpperCamelCase		)

						_A  ,		_A  ,		_A  ,		_A					  =			reshaped_hidden_states[0].shape
						_A					  =			(
						    reshaped_hidden_states[0].view(__UpperCamelCase    ,   __UpperCamelCase    ,   height * width		).permute(0    ,   2    ,   1		)
						)
						self.assertListEqual(
						    list(reshaped_hidden_states.shape[-2:]		)    ,   [num_patches, self.model_tester.embed_dim]    ,   )







		def  a_ (					self      :       Tuple		)       ->						str:





						'''simple docstring'''







						_A  ,		_A					  =			self.model_tester.prepare_config_and_inputs_for_common()

						_A					  =			(
						    self.model_tester.image_size
						    if isinstance(self.model_tester.image_size    ,   collections.abc.Iterable		)
						    else (self.model_tester.image_size, self.model_tester.image_size)
						)

						for model_class in self.all_model_classes[:-1]:
										_A					  =			True
										self.check_hidden_states_output(__UpperCamelCase    ,   __UpperCamelCase    ,   __UpperCamelCase    ,   __UpperCamelCase		)

										# check that output_hidden_states also work using config
										del inputs_dict["output_hidden_states"]
										_A					  =			True

										self.check_hidden_states_output(__UpperCamelCase    ,   __UpperCamelCase    ,   __UpperCamelCase    ,   __UpperCamelCase		)







		def  a_ (					self      :       Tuple		)       ->						str:





						'''simple docstring'''







						_A  ,		_A					  =			self.model_tester.prepare_config_and_inputs_for_common()
						_A					  =			3

						_A					  =			(
						    self.model_tester.image_size
						    if isinstance(self.model_tester.image_size    ,   collections.abc.Iterable		)
						    else (self.model_tester.image_size, self.model_tester.image_size)
						)
						_A					  =			(
						    config.patch_size
						    if isinstance(config.patch_size    ,   collections.abc.Iterable		)
						    else (config.patch_size, config.patch_size)
						)

						_A					  =			image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
						_A					  =			image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

						for model_class in self.all_model_classes[:-1]:
										_A					  =			True
										self.check_hidden_states_output(__UpperCamelCase    ,   __UpperCamelCase    ,   __UpperCamelCase    ,   (padded_height, padded_width)		)

										# check that output_hidden_states also work using config
										del inputs_dict["output_hidden_states"]
										_A					  =			True
										self.check_hidden_states_output(__UpperCamelCase    ,   __UpperCamelCase    ,   __UpperCamelCase    ,   (padded_height, padded_width)		)







		@slow
		def  a_ (					self      :       Optional[Any]		)       ->						str:





						'''simple docstring'''







						for model_name in FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										_A					  =			FocalNetModel.from_pretrained(__UpperCamelCase		)
										self.assertIsNotNone(__UpperCamelCase		)







		def  a_ (					self      :       Optional[Any]		)       ->						Dict:





						'''simple docstring'''







						_A  ,		_A					  =			self.model_tester.prepare_config_and_inputs_for_common()

						_A					  =			_config_zero_init(__UpperCamelCase		)
						for model_class in self.all_model_classes:
										_A					  =			model_class(config=__UpperCamelCase		)
										for name, param in model.named_parameters():
														if "embeddings" not in name and param.requires_grad:
																		self.assertIn(
																		    ((param.data.mean() * 1E9).round() / 1E9).item()    ,   [0.0, 1.0]    ,   msg=F"""Parameter {name} of model {model_class} seems not properly initialized"""    ,   )





@require_vision
@require_torch
class snake_case      (  unittest.TestCase):

		'''simple docstring'''





		@cached_property
		def  a_ (					self      :       Dict		)       ->						Tuple:





						'''simple docstring'''







						return AutoImageProcessor.from_pretrained("microsoft/focalnet-tiny"		) if is_vision_available() else None







		@slow
		def  a_ (					self      :       Optional[int]		)       ->						Optional[int]:





						'''simple docstring'''







						_A					  =			FocalNetForImageClassification.from_pretrained("microsoft/focalnet-tiny"		).to(__UpperCamelCase		)
						_A					  =			self.default_image_processor

						_A					  =			Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"		)
						_A					  =			image_processor(images=__UpperCamelCase    ,   return_tensors="pt"		).to(__UpperCamelCase		)

						# forward pass
						with torch.no_grad():
										_A					  =			model(**__UpperCamelCase		)

						# verify the logits
						_A					  =			torch.Size((1, 10_00)		)
						self.assertEqual(outputs.logits.shape    ,   __UpperCamelCase		)
						_A					  =			torch.tensor([0.2_1_6_6, -0.4_3_6_8, 0.2_1_9_1]		).to(__UpperCamelCase		)
						self.assertTrue(torch.allclose(outputs.logits[0, :3]    ,   __UpperCamelCase    ,   atol=1E-4		)		)
						self.assertTrue(outputs.logits.argmax(dim=-1		).item()    ,   2_81		)





@require_torch
class snake_case      (  _UpperCAmelCase     ,				unittest.TestCase):

		'''simple docstring'''





		__UpperCamelCase           =			(FocalNetBackbone,) if is_torch_available() else ()
		__UpperCamelCase           =			FocalNetConfig

		__UpperCamelCase           =			False







		def  a_ (					self      :       Tuple		)       ->						List[Any]:





						'''simple docstring'''







						_A					  =			FocalNetModelTester(self		)



"""simple docstring"""


import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
		from ...tokenization_utils_base import TextInput
from ...utils import logging


a_       							=    logging.get_logger(__name__)

a_       							=    {"vocab_file": "spiece.model"}

a_       							=    {
    "vocab_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/spiece.model",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/spiece.model",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/spiece.model",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/spiece.model",
    }
}


# TODO(PVP) - this should be removed in Transformers v5
a_       							=    {
    "t5-small": 5_12,
    "t5-base": 5_12,
    "t5-large": 5_12,
    "t5-3b": 5_12,
    "t5-11b": 5_12,
}

a_       							=    "▁"
class snake_case      (  _UpperCamelCase):
		__UpperCamelCase           =			VOCAB_FILES_NAMES
		__UpperCamelCase           =			PRETRAINED_VOCAB_FILES_MAP
		__UpperCamelCase           =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__UpperCamelCase           =			['input_ids', 'attention_mask']







		def __init__(					self      :       List[str]    ,   a__      :       Optional[int]    ,   a__      :       Union[str, Any]="</s>"    ,   a__      :       Union[str, Any]="<unk>"    ,   a__      :       str="<pad>"    ,   a__      :       Optional[int]=1_00    ,   a__      :       List[Any]=None    ,   a__      :       Optional[Dict[str, Any]] = None    ,   a__      :       Any=True    ,   **a__      :       Optional[int]    ,   )       ->						None:





						'''simple docstring'''







						if extra_ids > 0 and additional_special_tokens is None:
										_A					  =			[F"""<extra_id_{i}>""" for i in range(a__		)]
						elif extra_ids > 0 and additional_special_tokens is not None:
										# Check that we have the right number of extra_id special tokens
										_A					  =			len(set(filter(lambda a__		: bool("extra_id" in str(a__		)		)    ,   a__		)		)		)
										if extra_tokens != extra_ids:
														raise ValueError(
														    F"""Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"""
														    " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
														    " tokens"		)
						if legacy:
										logger.warning_once(
										    F"""You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"""
										    " read the related pull request available at https://github.com/huggingface/transformers/pull/24565"		)

						_A					  =			legacy
						_A					  =			{} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    eos_token=a__    ,   unk_token=a__    ,   pad_token=a__    ,   extra_ids=a__    ,   additional_special_tokens=a__    ,   sp_model_kwargs=self.sp_model_kwargs    ,   legacy=a__    ,   **a__    ,   )

						_A					  =			vocab_file
						_A					  =			extra_ids

						_A					  =			spm.SentencePieceProcessor(**self.sp_model_kwargs		)
						self.sp_model.Load(a__		)







		@staticmethod
		def  a_ (					a__      :       List[str]    ,   a__      :       Optional[int]    ,   a__      :       Tuple		)       ->						Tuple:





						'''simple docstring'''







						if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
										_A					  =			TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
										if init_max_model_length is not None and init_max_model_length != max_model_length:
														return init_max_model_length
										elif init_max_model_length is None:
														warnings.warn(
														    "This tokenizer was incorrectly instantiated with a model max length of"
														    F""" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"""
														    " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
														    " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
														    F""" {pretrained_model_name_or_path} automatically truncating your input to"""
														    F""" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"""
														    F""" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"""
														    " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
														    " instantiate this tokenizer with `model_max_length` set to your preferred value."    ,   a__    ,   )

						return max_model_length







		@property
		def  a_ (					self      :       List[Any]		)       ->						Dict:





						'''simple docstring'''







						return self.sp_model.get_piece_size() + self._extra_ids







		def  a_ (					self      :       Dict		)       ->						Optional[Any]:





						'''simple docstring'''







						_A					  =			{self.convert_ids_to_tokens(a__		): i for i in range(self.vocab_size		)}
						vocab.update(self.added_tokens_encoder		)
						return vocab







		def  a_ (					self      :       Optional[Any]    ,   a__      :       List[int]    ,   a__      :       Optional[List[int]] = None    ,   a__      :       bool = False		)       ->						List[int]:





						'''simple docstring'''







						if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=a__    ,   token_ids_a=a__    ,   already_has_special_tokens=a__		)

						# normal case: some special tokens
						if token_ids_a is None:
										return ([0] * len(a__		)) + [1]
						return ([0] * len(a__		)) + [1] + ([0] * len(a__		)) + [1]







		def  a_ (					self      :       List[str]		)       ->						List[str]:





						'''simple docstring'''







						return list(
						    set(filter(lambda a__		: bool(re.search(r"<extra_id_\d+>"    ,   a__		)		) is not None    ,   self.additional_special_tokens		)		)		)







		def  a_ (					self      :       str		)       ->						List[Any]:





						'''simple docstring'''







						return [self._convert_token_to_id(a__		) for token in self.get_sentinel_tokens()]







		def  a_ (					self      :       List[Any]    ,   a__      :       List[int]		)       ->						List[int]:





						'''simple docstring'''







						if len(a__		) > 0 and token_ids[-1] == self.eos_token_id:
										warnings.warn(
										    F"""This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"""
										    " eos tokens being added."		)
										return token_ids
						else:
										return token_ids + [self.eos_token_id]







		def  a_ (					self      :       int    ,   a__      :       List[int]    ,   a__      :       Optional[List[int]] = None		)       ->						List[int]:





						'''simple docstring'''







						_A					  =			[self.eos_token_id]

						if token_ids_a is None:
										return len(token_ids_a + eos		) * [0]
						return len(token_ids_a + eos + token_ids_a + eos		) * [0]







		def  a_ (					self      :       Union[str, Any]    ,   a__      :       List[int]    ,   a__      :       Optional[List[int]] = None		)       ->						List[int]:





						'''simple docstring'''







						_A					  =			self._add_eos_if_not_present(a__		)
						if token_ids_a is None:
										return token_ids_a
						else:
										_A					  =			self._add_eos_if_not_present(a__		)
										return token_ids_a + token_ids_a







		def __getstate__(					self      :       Dict		)       ->						Union[str, Any]:





						'''simple docstring'''







						_A					  =			self.__dict__.copy()
						_A					  =			None
						return state







		def __setstate__(					self      :       int    ,   a__      :       Optional[int]		)       ->						Union[str, Any]:





						'''simple docstring'''







						_A					  =			d

						# for backward compatibility
						if not hasattr(self    ,   "sp_model_kwargs"		):
										_A					  =			{}

						_A					  =			spm.SentencePieceProcessor(**self.sp_model_kwargs		)
						self.sp_model.Load(self.vocab_file		)







		def  a_ (					self      :       int    ,   a__      :       "TextInput"    ,   **a__      :       List[str]		)       ->						List[str]:





						'''simple docstring'''







						if not self.legacy:
										_A					  =			SPIECE_UNDERLINE + text.replace(a__    ,   " "		)
						return super().tokenize(a__    ,   **a__		)







		def  a_ (					self      :       str    ,   a__      :       Dict    ,   **a__      :       Optional[int]		)       ->						Any:





						'''simple docstring'''







						if not self.legacy:
										_A					  =			text.startswith(a__		)
										if is_first:
														_A					  =			text[1:]

						_A					  =			self.sp_model.encode(a__    ,   out_type=a__		)

						if not self.legacy and not is_first and not text.startswith(" "		) and tokens[0].startswith(a__		):
										_A					  =			([tokens[0][1:]] if len(tokens[0]		) > 1 else []) + tokens[1:]
						return tokens







		def  a_ (					self      :       int    ,   a__      :       List[Any]		)       ->						List[str]:





						'''simple docstring'''







						if token.startswith("<extra_id_"		):
										_A					  =			re.match(r"<extra_id_(\d+)>"    ,   a__		)
										_A					  =			int(match.group(1		)		)
										return self.vocab_size - num - 1
						return self.sp_model.piece_to_id(a__		)







		def  a_ (					self      :       Dict    ,   a__      :       Union[str, Any]		)       ->						Any:





						'''simple docstring'''







						if index < self.sp_model.get_piece_size():
										_A					  =			self.sp_model.IdToPiece(a__		)
						else:
										_A					  =			F"""<extra_id_{self.vocab_size - 1 - index}>"""
						return token







		def  a_ (					self      :       Optional[int]    ,   a__      :       Tuple		)       ->						List[str]:





						'''simple docstring'''







						_A					  =			[]
						_A					  =			""
						_A					  =			False
						for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
														if not prev_is_special:
																		out_string += " "
														out_string += self.sp_model.decode(a__		) + token
														_A					  =			True
														_A					  =			[]
										else:
														current_sub_tokens.append(a__		)
														_A					  =			False
						out_string += self.sp_model.decode(a__		)
						return out_string.strip()







		def  a_ (					self      :       Dict    ,   a__      :       str    ,   a__      :       Optional[str] = None		)       ->						Tuple[str]:





						'''simple docstring'''







						if not os.path.isdir(a__		):
										logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""		)
										return
						_A					  =			os.path.join(
						    a__    ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

						if os.path.abspath(self.vocab_file		) != os.path.abspath(a__		) and os.path.isfile(self.vocab_file		):
										copyfile(self.vocab_file    ,   a__		)
						elif not os.path.isfile(self.vocab_file		):
										with open(a__    ,   "wb"		) as fi:
														_A					  =			self.sp_model.serialized_model_proto()
														fi.write(a__		)

						return (out_vocab_file,)