


"""simple docstring"""


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_       							=    logging.get_logger(__name__)
a_       							=    {"vocab_file": "spiece.model"}

a_       							=    {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

a_       							=    {
    "albert-base-v1": 5_12,
    "albert-large-v1": 5_12,
    "albert-xlarge-v1": 5_12,
    "albert-xxlarge-v1": 5_12,
    "albert-base-v2": 5_12,
    "albert-large-v2": 5_12,
    "albert-xlarge-v2": 5_12,
    "albert-xxlarge-v2": 5_12,
}

a_       							=    "▁"
class snake_case      (  lowercase__):
		__UpperCamelCase           =			VOCAB_FILES_NAMES
		__UpperCamelCase           =			PRETRAINED_VOCAB_FILES_MAP
		__UpperCamelCase           =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







		def __init__(					self      :       int    ,   a__      :       List[str]    ,   a__      :       str=True    ,   a__      :       Union[str, Any]=True    ,   a__      :       int=False    ,   a__      :       Tuple="[CLS]"    ,   a__      :       Any="[SEP]"    ,   a__      :       Union[str, Any]="<unk>"    ,   a__      :       Union[str, Any]="[SEP]"    ,   a__      :       Optional[Any]="<pad>"    ,   a__      :       str="[CLS]"    ,   a__      :       Dict="[MASK]"    ,   a__      :       Optional[Dict[str, Any]] = None    ,   **a__      :       Optional[int]    ,   )       ->						str:





						'''simple docstring'''







						_A					  =			(
						    AddedToken(UpperCAmelCase__    ,   lstrip=UpperCAmelCase__    ,   rstrip=UpperCAmelCase__    ,   normalized=UpperCAmelCase__		)
						    if isinstance(UpperCAmelCase__    ,   UpperCAmelCase__		)
						    else mask_token
						)

						_A					  =			{} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    do_lower_case=UpperCAmelCase__    ,   remove_space=UpperCAmelCase__    ,   keep_accents=UpperCAmelCase__    ,   bos_token=UpperCAmelCase__    ,   eos_token=UpperCAmelCase__    ,   unk_token=UpperCAmelCase__    ,   sep_token=UpperCAmelCase__    ,   pad_token=UpperCAmelCase__    ,   cls_token=UpperCAmelCase__    ,   mask_token=UpperCAmelCase__    ,   sp_model_kwargs=self.sp_model_kwargs    ,   **UpperCAmelCase__    ,   )

						_A					  =			do_lower_case
						_A					  =			remove_space
						_A					  =			keep_accents
						_A					  =			vocab_file

						_A					  =			spm.SentencePieceProcessor(**self.sp_model_kwargs		)
						self.sp_model.Load(UpperCAmelCase__		)







		@property
		def  a_ (					self      :       Optional[Any]		)       ->						List[str]:





						'''simple docstring'''







						return len(self.sp_model		)







		def  a_ (					self      :       Optional[int]		)       ->						Union[str, Any]:





						'''simple docstring'''







						_A					  =			{self.convert_ids_to_tokens(UpperCAmelCase__		): i for i in range(self.vocab_size		)}
						vocab.update(self.added_tokens_encoder		)
						return vocab







		def __getstate__(					self      :       Optional[Any]		)       ->						List[str]:





						'''simple docstring'''







						_A					  =			self.__dict__.copy()
						_A					  =			None
						return state







		def __setstate__(					self      :       Tuple    ,   a__      :       Optional[Any]		)       ->						List[Any]:





						'''simple docstring'''







						_A					  =			d

						# for backward compatibility
						if not hasattr(self    ,   "sp_model_kwargs"		):
										_A					  =			{}

						_A					  =			spm.SentencePieceProcessor(**self.sp_model_kwargs		)
						self.sp_model.Load(self.vocab_file		)







		def  a_ (					self      :       Any    ,   a__      :       Optional[Any]		)       ->						str:





						'''simple docstring'''







						if self.remove_space:
										_A					  =			''' '''.join(inputs.strip().split()		)
						else:
										_A					  =			inputs
						_A					  =			outputs.replace("``"    ,   "\""		).replace("\'\'"    ,   "\""		)

						if not self.keep_accents:
										_A					  =			unicodedata.normalize("NFKD"    ,   UpperCAmelCase__		)
										_A					  =			''''''.join([c for c in outputs if not unicodedata.combining(UpperCAmelCase__		)]		)
						if self.do_lower_case:
										_A					  =			outputs.lower()

						return outputs







		def  a_ (					self      :       str    ,   a__      :       str		)       ->						Any:





						'''simple docstring'''







						_A					  =			self.preprocess_text(UpperCAmelCase__		)
						_A					  =			self.sp_model.encode(UpperCAmelCase__    ,   out_type=UpperCAmelCase__		)
						_A					  =			[]
						for piece in pieces:
										if len(UpperCAmelCase__		) > 1 and piece[-1] == str(","		) and piece[-2].isdigit():
														_A					  =			self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase__    ,   ""		)		)
														if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																		if len(cur_pieces[0]		) == 1:
																						_A					  =			cur_pieces[1:]
																		else:
																						_A					  =			cur_pieces[0][1:]
														cur_pieces.append(piece[-1]		)
														new_pieces.extend(UpperCAmelCase__		)
										else:
														new_pieces.append(UpperCAmelCase__		)

						return new_pieces







		def  a_ (					self      :       Tuple    ,   a__      :       Dict		)       ->						Optional[Any]:





						'''simple docstring'''







						return self.sp_model.PieceToId(UpperCAmelCase__		)







		def  a_ (					self      :       Any    ,   a__      :       List[Any]		)       ->						Union[str, Any]:





						'''simple docstring'''







						return self.sp_model.IdToPiece(UpperCAmelCase__		)







		def  a_ (					self      :       int    ,   a__      :       List[Any]		)       ->						Dict:





						'''simple docstring'''







						_A					  =			[]
						_A					  =			''''''
						_A					  =			False
						for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
														if not prev_is_special:
																		out_string += " "
														out_string += self.sp_model.decode(UpperCAmelCase__		) + token
														_A					  =			True
														_A					  =			[]
										else:
														current_sub_tokens.append(UpperCAmelCase__		)
														_A					  =			False
						out_string += self.sp_model.decode(UpperCAmelCase__		)
						return out_string.strip()







		def  a_ (					self      :       List[str]    ,   a__      :       List[int]    ,   a__      :       Optional[List[int]] = None		)       ->						List[str]:





						'''simple docstring'''







						_A					  =			[self.sep_token_id]
						_A					  =			[self.cls_token_id]
						if token_ids_a is None:
										return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep







		def  a_ (					self      :       List[Any]    ,   a__      :       List[int]    ,   a__      :       Optional[List[int]] = None    ,   a__      :       bool = False		)       ->						List[str]:





						'''simple docstring'''







						if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=UpperCAmelCase__    ,   token_ids_a=UpperCAmelCase__    ,   already_has_special_tokens=UpperCAmelCase__		)

						if token_ids_a is not None:
										return [1] + ([0] * len(UpperCAmelCase__		)) + [1] + ([0] * len(UpperCAmelCase__		)) + [1]
						return [1] + ([0] * len(UpperCAmelCase__		)) + [1]







		def  a_ (					self      :       Optional[int]    ,   a__      :       List[int]    ,   a__      :       Optional[List[int]] = None		)       ->						List[Any]:





						'''simple docstring'''







						_A					  =			[self.sep_token_id]
						_A					  =			[self.cls_token_id]

						if token_ids_a is None:
										return len(cls + token_ids_a + sep		) * [0]
						return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







		def  a_ (					self      :       Any    ,   a__      :       str    ,   a__      :       Optional[str] = None		)       ->						List[Any]:





						'''simple docstring'''







						if not os.path.isdir(UpperCAmelCase__		):
										logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""		)
										return
						_A					  =			os.path.join(
						    UpperCAmelCase__    ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

						if os.path.abspath(self.vocab_file		) != os.path.abspath(UpperCAmelCase__		) and os.path.isfile(self.vocab_file		):
										copyfile(self.vocab_file    ,   UpperCAmelCase__		)
						elif not os.path.isfile(self.vocab_file		):
										with open(UpperCAmelCase__    ,   "wb"		) as fi:
														_A					  =			self.sp_model.serialized_model_proto()
														fi.write(UpperCAmelCase__		)

						return (out_vocab_file,)



"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a_       							=    {
    "configuration_upernet": ["UperNetConfig"],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		a_       							=    [
		    "UperNetForSemanticSegmentation",
		    "UperNetPreTrainedModel",
		]


if TYPE_CHECKING:
		from .configuration_upernet import UperNetConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_upernet import UperNetForSemanticSegmentation, UperNetPreTrainedModel


else:
		import sys

		a_       							=    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)