

import collections
import inspect
import unittest

from transformers import FocalNetConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch
							from torch import nn

							from transformers import (
							    FocalNetBackbone,
							    FocalNetForImageClassification,
							    FocalNetForMaskedImageModeling,
							    FocalNetModel,
							)
							from transformers.models.focalnet.modeling_focalnet import FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
							from PIL import Image

							from transformers import AutoImageProcessor







class 	lowercase    :


		"""simple docstring"""




		def __init__(				self	,       __snake_case	,       __snake_case=13	,       __snake_case=32	,       __snake_case=2	,       __snake_case=3	,       __snake_case=16	,       __snake_case=[32, 64, 1_28]	,       __snake_case=[1, 2, 1]	,       __snake_case=[2, 2, 4]	,       __snake_case=2	,       __snake_case=2.0	,       __snake_case=True	,       __snake_case=0.0	,       __snake_case=0.0	,       __snake_case=0.1	,       __snake_case="gelu"	,       __snake_case=False	,       __snake_case=True	,       __snake_case=0.0_2	,       __snake_case=1e-5	,       __snake_case=True	,       __snake_case=None	,       __snake_case=True	,       __snake_case=10	,       __snake_case=8	,       __snake_case=["stage1", "stage2"]	,       __snake_case=[1, 2]	,       ):
			_UpperCamelCase   :		Optional[int]			=						parent
			_UpperCamelCase   :		List[str]			=						batch_size
			_UpperCamelCase   :		Union[str, Any]			=						image_size
			_UpperCamelCase   :		List[Any]			=						patch_size
			_UpperCamelCase   :		Dict			=						num_channels
			_UpperCamelCase   :		List[Any]			=						embed_dim
			_UpperCamelCase   :		Tuple			=						hidden_sizes
			_UpperCamelCase   :		int			=						depths
			_UpperCamelCase   :		int			=						num_heads
			_UpperCamelCase   :		Optional[int]			=						window_size
			_UpperCamelCase   :		List[Any]			=						mlp_ratio
			_UpperCamelCase   :		Optional[int]			=						qkv_bias
			_UpperCamelCase   :		str			=						hidden_dropout_prob
			_UpperCamelCase   :		Any			=						attention_probs_dropout_prob
			_UpperCamelCase   :		str			=						drop_path_rate
			_UpperCamelCase   :		Union[str, Any]			=						hidden_act
			_UpperCamelCase   :		Union[str, Any]			=						use_absolute_embeddings
			_UpperCamelCase   :		Optional[Any]			=						patch_norm
			_UpperCamelCase   :		Tuple			=						layer_norm_eps
			_UpperCamelCase   :		Tuple			=						initializer_range
			_UpperCamelCase   :		str			=						is_training
			_UpperCamelCase   :		int			=						scope
			_UpperCamelCase   :		Union[str, Any]			=						use_labels
			_UpperCamelCase   :		str			=						type_sequence_label_size
			_UpperCamelCase   :		Any			=						encoder_stride
			_UpperCamelCase   :		str			=						out_features
			_UpperCamelCase   :		Optional[Any]			=						out_indices


		def 		A__					(				self):
			_UpperCamelCase   :		List[str]			=						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

			_UpperCamelCase   :		List[str]			=						None
			if self.use_labels:
				_UpperCamelCase   :		List[Any]			=						ids_tensor([self.batch_size]	,       self.type_sequence_label_size)

			_UpperCamelCase   :		Union[str, Any]			=						self.get_config()

			return config, pixel_values, labels


		def 		A__					(				self):
			return FocalNetConfig(
			    image_size=self.image_size	,       patch_size=self.patch_size	,       num_channels=self.num_channels	,       embed_dim=self.embed_dim	,       hidden_sizes=self.hidden_sizes	,       depths=self.depths	,       num_heads=self.num_heads	,       window_size=self.window_size	,       mlp_ratio=self.mlp_ratio	,       qkv_bias=self.qkv_bias	,       hidden_dropout_prob=self.hidden_dropout_prob	,       attention_probs_dropout_prob=self.attention_probs_dropout_prob	,       drop_path_rate=self.drop_path_rate	,       hidden_act=self.hidden_act	,       use_absolute_embeddings=self.use_absolute_embeddings	,       path_norm=self.patch_norm	,       layer_norm_eps=self.layer_norm_eps	,       initializer_range=self.initializer_range	,       encoder_stride=self.encoder_stride	,       out_features=self.out_features	,       out_indices=self.out_indices	,       )


		def 		A__					(				self	,       __snake_case	,       __snake_case	,       __snake_case):
			_UpperCamelCase   :		Optional[Any]			=						FocalNetModel(config=__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase   :		Union[str, Any]			=						model(__snake_case)

			_UpperCamelCase   :		str			=						((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths) - 1))
			_UpperCamelCase   :		Any			=						int(config.embed_dim * 2 ** (len(config.depths) - 1))

			self.parent.assertEqual(result.last_hidden_state.shape	,       (self.batch_size, expected_seq_len, expected_dim))


		def 		A__					(				self	,       __snake_case	,       __snake_case	,       __snake_case):
			_UpperCamelCase   :		Union[str, Any]			=						FocalNetBackbone(config=__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase   :		str			=						model(__snake_case)

			# verify feature maps
			self.parent.assertEqual(len(result.feature_maps)	,       len(config.out_features))
			self.parent.assertListEqual(list(result.feature_maps[0].shape)	,       [self.batch_size, self.image_size, 8, 8])

			# verify channels
			self.parent.assertEqual(len(model.channels)	,       len(config.out_features))
			self.parent.assertListEqual(model.channels	,       config.hidden_sizes[:-1])

			# verify backbone works with out_features=None
			_UpperCamelCase   :		List[Any]			=						None
			_UpperCamelCase   :		Optional[Any]			=						FocalNetBackbone(config=__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase   :		Tuple			=						model(__snake_case)

			# verify feature maps
			self.parent.assertEqual(len(result.feature_maps)	,       1)
			self.parent.assertListEqual(list(result.feature_maps[0].shape)	,       [self.batch_size, self.image_size * 2, 4, 4])

			# verify channels
			self.parent.assertEqual(len(model.channels)	,       1)
			self.parent.assertListEqual(model.channels	,       [config.hidden_sizes[-1]])


		def 		A__					(				self	,       __snake_case	,       __snake_case	,       __snake_case):
			_UpperCamelCase   :		List[Any]			=						FocalNetForMaskedImageModeling(config=__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase   :		Any			=						model(__snake_case)
			self.parent.assertEqual(
			    result.reconstruction.shape	,       (self.batch_size, self.num_channels, self.image_size, self.image_size))

			# test greyscale images
			_UpperCamelCase   :		Optional[int]			=						1
			_UpperCamelCase   :		Any			=						FocalNetForMaskedImageModeling(__snake_case)
			model.to(__snake_case)
			model.eval()

			_UpperCamelCase   :		Optional[int]			=						floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
			_UpperCamelCase   :		str			=						model(__snake_case)
			self.parent.assertEqual(result.reconstruction.shape	,       (self.batch_size, 1, self.image_size, self.image_size))


		def 		A__					(				self	,       __snake_case	,       __snake_case	,       __snake_case):
			_UpperCamelCase   :		int			=						self.type_sequence_label_size
			_UpperCamelCase   :		Dict			=						FocalNetForImageClassification(__snake_case)
			model.to(__snake_case)
			model.eval()
			_UpperCamelCase   :		Tuple			=						model(__snake_case	,       labels=__snake_case)
			self.parent.assertEqual(result.logits.shape	,       (self.batch_size, self.type_sequence_label_size))

			# test greyscale images
			_UpperCamelCase   :		Dict			=						1
			_UpperCamelCase   :		Optional[int]			=						FocalNetForImageClassification(__snake_case)
			model.to(__snake_case)
			model.eval()

			_UpperCamelCase   :		Optional[int]			=						floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
			_UpperCamelCase   :		Optional[int]			=						model(__snake_case)
			self.parent.assertEqual(result.logits.shape	,       (self.batch_size, self.type_sequence_label_size))






		def 		A__					(				self):
			_UpperCamelCase   :		Any			=						self.prepare_config_and_inputs()

			_UpperCamelCase	,		_UpperCamelCase	,		_UpperCamelCase   :		Union[str, Any]			=						config_and_inputs
			_UpperCamelCase   :		int			=						{'pixel_values': pixel_values}
			return config, inputs_dict







@require_torch
class 	lowercase    (				_lowercase    ,							_lowercase    ,							unittest.TestCase							):


		"""simple docstring"""




		a__					       =  (
		    (
		        FocalNetModel,
		        FocalNetForImageClassification,
		        FocalNetForMaskedImageModeling,
		        FocalNetBackbone,
		    )
		    if is_torch_available()
		    else ()
		)
		a__					       =  (
		    {"feature-extraction": FocalNetModel, "image-classification": FocalNetForImageClassification}
		    if is_torch_available()
		    else {}
		)
		a__					       =  False

		a__					       =  False
		a__					       =  False
		a__					       =  False
		a__					       =  False


		def 		A__					(				self):
			_UpperCamelCase   :		Optional[int]			=						FocalNetModelTester(self)
			_UpperCamelCase   :		Tuple			=						ConfigTester(self	,       config_class=__snake_case	,       embed_dim=37	,       has_text_modality=__snake_case)


		def 		A__					(				self):
			self.create_and_test_config_common_properties()
			self.config_tester.create_and_test_config_to_json_string()
			self.config_tester.create_and_test_config_to_json_file()
			self.config_tester.create_and_test_config_from_and_save_pretrained()
			self.config_tester.create_and_test_config_with_num_labels()
			self.config_tester.check_config_can_be_init_without_params()
			self.config_tester.check_config_arguments_init()


		def 		A__					(				self):
			return


		def 		A__					(				self):
			_UpperCamelCase   :		Any			=						self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*__snake_case)


		def 		A__					(				self):
			_UpperCamelCase   :		List[str]			=						self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_backbone(*__snake_case)


		def 		A__					(				self):
			_UpperCamelCase   :		int			=						self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_masked_image_modeling(*__snake_case)


		def 		A__					(				self):
			_UpperCamelCase   :		str			=						self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_image_classification(*__snake_case)


		@unittest.skip(reason='FocalNet does not use inputs_embeds')
		def 		A__					(				self):
			pass


		@unittest.skip(reason='FocalNet does not use feedforward chunking')
		def 		A__					(				self):
			pass


		def 		A__					(				self):
			_UpperCamelCase	,		_UpperCamelCase   :		Union[str, Any]			=						self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes[:-1]:
				_UpperCamelCase   :		int			=						model_class(__snake_case)
				self.assertIsInstance(model.get_input_embeddings()	,       (nn.Module))
				_UpperCamelCase   :		Optional[Any]			=						model.get_output_embeddings()
				self.assertTrue(x is None or isinstance(__snake_case	,       nn.Linear))


		def 		A__					(				self):
			_UpperCamelCase	,		_UpperCamelCase   :		str			=						self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes[:-1]:
				_UpperCamelCase   :		Tuple			=						model_class(__snake_case)
				_UpperCamelCase   :		str			=						inspect.signature(model.forward)
				# signature.parameters is an OrderedDict => so arg_names order is deterministic
				_UpperCamelCase   :		str			=						[*signature.parameters.keys()]

				_UpperCamelCase   :		List[Any]			=						['pixel_values']
				self.assertListEqual(arg_names[:1]	,       __snake_case)


		def 		A__					(				self	,       __snake_case	,       __snake_case	,       __snake_case	,       __snake_case):
			_UpperCamelCase   :		Union[str, Any]			=						model_class(__snake_case)
			model.to(__snake_case)
			model.eval()

			with torch.no_grad():
				_UpperCamelCase   :		Any			=						model(**self._prepare_for_class(__snake_case	,       __snake_case))

			_UpperCamelCase   :		Tuple			=						outputs.hidden_states

			_UpperCamelCase   :		Dict			=						getattr(
			    self.model_tester	,       'expected_num_hidden_layers'	,       len(self.model_tester.depths) + 1)
			self.assertEqual(len(__snake_case)	,       __snake_case)

			# FocalNet has a different seq_length
			_UpperCamelCase   :		Optional[Any]			=						(
			    config.patch_size
			    if isinstance(config.patch_size	,       collections.abc.Iterable)
			    else (config.patch_size, config.patch_size)
			)

			_UpperCamelCase   :		Union[str, Any]			=						(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

			self.assertListEqual(
			    list(hidden_states[0].shape[-2:])	,       [num_patches, self.model_tester.embed_dim]	,       )

			_UpperCamelCase   :		List[str]			=						outputs.reshaped_hidden_states
			self.assertEqual(len(__snake_case)	,       __snake_case)

			_UpperCamelCase	,		_UpperCamelCase	,		_UpperCamelCase	,		_UpperCamelCase   :		Union[str, Any]			=						reshaped_hidden_states[0].shape
			_UpperCamelCase   :		int			=						(
			    reshaped_hidden_states[0].view(__snake_case	,       __snake_case	,       height * width).permute(0	,       2	,       1)
			)
			self.assertListEqual(
			    list(reshaped_hidden_states.shape[-2:])	,       [num_patches, self.model_tester.embed_dim]	,       )


		def 		A__					(				self):
			_UpperCamelCase	,		_UpperCamelCase   :		str			=						self.model_tester.prepare_config_and_inputs_for_common()

			_UpperCamelCase   :		List[str]			=						(
			    self.model_tester.image_size
			    if isinstance(self.model_tester.image_size	,       collections.abc.Iterable)
			    else (self.model_tester.image_size, self.model_tester.image_size)
			)

			for model_class in self.all_model_classes[:-1]:
				_UpperCamelCase   :		List[str]			=						True
				self.check_hidden_states_output(__snake_case	,       __snake_case	,       __snake_case	,       __snake_case)

				# check that output_hidden_states also work using config
				del inputs_dict["output_hidden_states"]
				_UpperCamelCase   :		str			=						True

				self.check_hidden_states_output(__snake_case	,       __snake_case	,       __snake_case	,       __snake_case)


		def 		A__					(				self):
			_UpperCamelCase	,		_UpperCamelCase   :		Dict			=						self.model_tester.prepare_config_and_inputs_for_common()
			_UpperCamelCase   :		str			=						3

			_UpperCamelCase   :		Optional[Any]			=						(
			    self.model_tester.image_size
			    if isinstance(self.model_tester.image_size	,       collections.abc.Iterable)
			    else (self.model_tester.image_size, self.model_tester.image_size)
			)
			_UpperCamelCase   :		Dict			=						(
			    config.patch_size
			    if isinstance(config.patch_size	,       collections.abc.Iterable)
			    else (config.patch_size, config.patch_size)
			)

			_UpperCamelCase   :		Optional[int]			=						image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
			_UpperCamelCase   :		Union[str, Any]			=						image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

			for model_class in self.all_model_classes[:-1]:
				_UpperCamelCase   :		List[str]			=						True
				self.check_hidden_states_output(__snake_case	,       __snake_case	,       __snake_case	,       (padded_height, padded_width))

				# check that output_hidden_states also work using config
				del inputs_dict["output_hidden_states"]
				_UpperCamelCase   :		List[Any]			=						True
				self.check_hidden_states_output(__snake_case	,       __snake_case	,       __snake_case	,       (padded_height, padded_width))


		@slow
		def 		A__					(				self):
			for model_name in FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
				_UpperCamelCase   :		int			=						FocalNetModel.from_pretrained(__snake_case)
				self.assertIsNotNone(__snake_case)






		def 		A__					(				self):
			_UpperCamelCase	,		_UpperCamelCase   :		Optional[Any]			=						self.model_tester.prepare_config_and_inputs_for_common()

			_UpperCamelCase   :		List[str]			=						_config_zero_init(__snake_case)
			for model_class in self.all_model_classes:
				_UpperCamelCase   :		Dict			=						model_class(config=__snake_case)
				for name, param in model.named_parameters():
					if "embeddings" not in name and param.requires_grad:
						self.assertIn(
						    ((param.data.mean() * 1e9).round() / 1e9).item()	,       [0.0, 1.0]	,       msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''	,       )







@require_vision
@require_torch
class 	lowercase    (				unittest.TestCase							):


		"""simple docstring"""




		@cached_property
		def 		A__					(				self):
			# TODO update organization
			return AutoImageProcessor.from_pretrained('microsoft/focalnet-tiny') if is_vision_available() else None






		@slow
		def 		A__					(				self):
			_UpperCamelCase   :		int			=						FocalNetForImageClassification.from_pretrained('microsoft/focalnet-tiny').to(__snake_case)
			_UpperCamelCase   :		List[str]			=						self.default_image_processor

			_UpperCamelCase   :		int			=						Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png')
			_UpperCamelCase   :		Union[str, Any]			=						image_processor(images=__snake_case	,       return_tensors='pt').to(__snake_case)

			# forward pass
			with torch.no_grad():
				_UpperCamelCase   :		Optional[int]			=						model(**__snake_case)

			# verify the logits
			_UpperCamelCase   :		Union[str, Any]			=						torch.Size((1, 10_00))
			self.assertEqual(outputs.logits.shape	,       __snake_case)
			_UpperCamelCase   :		Tuple			=						torch.tensor([0.2_1_6_6, -0.4_3_6_8, 0.2_1_9_1]).to(__snake_case)
			self.assertTrue(torch.allclose(outputs.logits[0, :3]	,       __snake_case	,       atol=1e-4))
			self.assertTrue(outputs.logits.argmax(dim=-1).item()	,       2_81)







@require_torch
class 	lowercase    (				_lowercase    ,							unittest.TestCase							):


		"""simple docstring"""




		a__					       =  (FocalNetBackbone,) if is_torch_available() else ()
		a__					       =  FocalNetConfig

		a__					       =  False


		def 		A__					(				self):
			_UpperCamelCase   :		List[Any]			=						FocalNetModelTester(self)







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__								=		logging.get_logger(__name__)

lowerCAmelCase__								=		"""▁"""

lowerCAmelCase__								=		{"""vocab_file""": """sentencepiece.bpe.model"""}

lowerCAmelCase__								=		{
    """vocab_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

lowerCAmelCase__								=		{
    """xlm-roberta-base""": 5_1_2,
    """xlm-roberta-large""": 5_1_2,
    """xlm-roberta-large-finetuned-conll02-dutch""": 5_1_2,
    """xlm-roberta-large-finetuned-conll02-spanish""": 5_1_2,
    """xlm-roberta-large-finetuned-conll03-english""": 5_1_2,
    """xlm-roberta-large-finetuned-conll03-german""": 5_1_2,
}







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		a__					       =  VOCAB_FILES_NAMES
		a__					       =  PRETRAINED_VOCAB_FILES_MAP
		a__					       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a__					       =  ["input_ids", "attention_mask"]


		def __init__(				self	,       __snake_case	,       __snake_case="<s>"	,       __snake_case="</s>"	,       __snake_case="</s>"	,       __snake_case="<s>"	,       __snake_case="<unk>"	,       __snake_case="<pad>"	,       __snake_case="<mask>"	,       __snake_case = None	,       **__snake_case	,       ):
			# Mask token behave like a normal word, i.e. include the space before it
			_UpperCamelCase   :		Dict			=						AddedToken(__snake_case	,       lstrip=__snake_case	,       rstrip=__snake_case) if isinstance(__snake_case	,       __snake_case) else mask_token

			_UpperCamelCase   :		str			=						{} if sp_model_kwargs is None else sp_model_kwargs

			super().__init__(
			    bos_token=__snake_case	,       eos_token=__snake_case	,       unk_token=__snake_case	,       sep_token=__snake_case	,       cls_token=__snake_case	,       pad_token=__snake_case	,       mask_token=__snake_case	,       sp_model_kwargs=self.sp_model_kwargs	,       **__snake_case	,       )

			_UpperCamelCase   :		Union[str, Any]			=						spm.SentencePieceProcessor(**self.sp_model_kwargs)
			self.sp_model.Load(str(__snake_case))
			_UpperCamelCase   :		Dict			=						vocab_file

			# Original fairseq vocab and spm vocab must be "aligned":
			# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
			# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
			# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
			# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

			# Mimic fairseq token-to-id alignment for the first 4 token
			_UpperCamelCase   :		int			=						{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

			# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
			_UpperCamelCase   :		List[Any]			=						1

			_UpperCamelCase   :		Any			=						len(self.sp_model) + self.fairseq_offset
			_UpperCamelCase   :		List[str]			=						{v: k for k, v in self.fairseq_tokens_to_ids.items()}


		def __getstate__(				self):
			_UpperCamelCase   :		List[Any]			=						self.__dict__.copy()
			_UpperCamelCase   :		Optional[Any]			=						None
			_UpperCamelCase   :		Any			=						self.sp_model.serialized_model_proto()
			return state


		def __setstate__(				self	,       __snake_case):
			_UpperCamelCase   :		int			=						d

			# for backward compatibility
			if not hasattr(self	,       'sp_model_kwargs'):
				_UpperCamelCase   :		Tuple			=						{}

			_UpperCamelCase   :		Optional[int]			=						spm.SentencePieceProcessor(**self.sp_model_kwargs)
			self.sp_model.LoadFromSerializedProto(self.sp_model_proto)


		def 		A__					(				self	,       __snake_case	,       __snake_case = None):

			if token_ids_a is None:
				return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
			_UpperCamelCase   :		Tuple			=						[self.cls_token_id]
			_UpperCamelCase   :		int			=						[self.sep_token_id]
			return cls + token_ids_a + sep + sep + token_ids_a + sep


		def 		A__					(				self	,       __snake_case	,       __snake_case = None	,       __snake_case = False):

			if already_has_special_tokens:
				return super().get_special_tokens_mask(
				    token_ids_a=__snake_case	,       token_ids_a=__snake_case	,       already_has_special_tokens=__snake_case)

			if token_ids_a is None:
				return [1] + ([0] * len(__snake_case)) + [1]
			return [1] + ([0] * len(__snake_case)) + [1, 1] + ([0] * len(__snake_case)) + [1]


		def 		A__					(				self	,       __snake_case	,       __snake_case = None):
			_UpperCamelCase   :		Optional[Any]			=						[self.sep_token_id]
			_UpperCamelCase   :		Optional[Any]			=						[self.cls_token_id]

			if token_ids_a is None:
				return len(cls + token_ids_a + sep) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


		@property
		def 		A__					(				self):
			return len(self.sp_model) + self.fairseq_offset + 1  # Add the <mask> token


		def 		A__					(				self):
			_UpperCamelCase   :		List[str]			=						{self.convert_ids_to_tokens(__snake_case): i for i in range(self.vocab_size)}
			vocab.update(self.added_tokens_encoder)
			return vocab


		def 		A__					(				self	,       __snake_case):
			return self.sp_model.encode(__snake_case	,       out_type=__snake_case)


		def 		A__					(				self	,       __snake_case):
			if token in self.fairseq_tokens_to_ids:
				return self.fairseq_tokens_to_ids[token]
			_UpperCamelCase   :		str			=						self.sp_model.PieceToId(__snake_case)

			# Need to return unknown token if the SP model returned 0
			return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


		def 		A__					(				self	,       __snake_case):
			if index in self.fairseq_ids_to_tokens:
				return self.fairseq_ids_to_tokens[index]
			return self.sp_model.IdToPiece(index - self.fairseq_offset)


		def 		A__					(				self	,       __snake_case):
			_UpperCamelCase   :		Optional[int]			=						''.join(__snake_case).replace(__snake_case	,       ' ').strip()
			return out_string






		def 		A__					(				self	,       __snake_case	,       __snake_case = None):
			if not os.path.isdir(__snake_case):
				logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
				return
			_UpperCamelCase   :		str			=						os.path.join(
			    __snake_case	,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

			if os.path.abspath(self.vocab_file) != os.path.abspath(__snake_case) and os.path.isfile(self.vocab_file):
				copyfile(self.vocab_file	,       __snake_case)
			elif not os.path.isfile(self.vocab_file):
				with open(__snake_case	,       'wb') as fi:
					_UpperCamelCase   :		Any			=						self.sp_model.serialized_model_proto()
					fi.write(__snake_case)

			return (out_vocab_file,)




