


"""simple docstring"""


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_       							=    logging.get_logger(__name__)

a_       							=    "▁"

a_       							=    {"vocab_file": "sentencepiece.bpe.model", "monolingual_vocab_file": "dict.txt"}

a_       							=    {
    "vocab_file": {
        "vinai/bartpho-syllable": "https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model",
    },
    "monolingual_vocab_file": {
        "vinai/bartpho-syllable": "https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt",
    },
}

a_       							=    {"vinai/bartpho-syllable": 10_24}
class snake_case      (  _UpperCamelCase):
		__UpperCamelCase           =			VOCAB_FILES_NAMES
		__UpperCamelCase           =			PRETRAINED_VOCAB_FILES_MAP
		__UpperCamelCase           =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__UpperCamelCase           =			['input_ids', 'attention_mask']







		def __init__(					self      :       int    ,   a__      :       Optional[int]    ,   a__      :       str    ,   a__      :       Tuple="<s>"    ,   a__      :       int="</s>"    ,   a__      :       Any="</s>"    ,   a__      :       int="<s>"    ,   a__      :       Tuple="<unk>"    ,   a__      :       List[str]="<pad>"    ,   a__      :       Tuple="<mask>"    ,   a__      :       Optional[Dict[str, Any]] = None    ,   **a__      :       int    ,   )       ->						None:





						'''simple docstring'''







						_A					  =			AddedToken(a__    ,   lstrip=a__    ,   rstrip=a__		) if isinstance(a__    ,   a__		) else mask_token

						_A					  =			{} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    bos_token=a__    ,   eos_token=a__    ,   unk_token=a__    ,   sep_token=a__    ,   cls_token=a__    ,   pad_token=a__    ,   mask_token=a__    ,   sp_model_kwargs=self.sp_model_kwargs    ,   **a__    ,   )

						_A					  =			vocab_file
						_A					  =			monolingual_vocab_file
						_A					  =			spm.SentencePieceProcessor(**self.sp_model_kwargs		)
						self.sp_model.Load(str(a__		)		)

						# Load the reduced vocab

						# Keep order of special tokens for backward compatibility
						_A					  =			{}
						_A					  =			0
						for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
										if str(a__		) not in self.fairseq_tokens_to_ids:
														_A					  =			cnt
														cnt += 1
						with open(a__    ,   "r"    ,   encoding="utf-8"		) as f:
										for line in f.readlines():
														_A					  =			line.strip().split()[0]
														_A					  =			len(self.fairseq_tokens_to_ids		)
						if str(a__		) not in self.fairseq_tokens_to_ids:
										_A					  =			len(self.fairseq_tokens_to_ids		)

						_A					  =			{v: k for k, v in self.fairseq_tokens_to_ids.items()}







		def __getstate__(					self      :       Union[str, Any]		)       ->						Tuple:





						'''simple docstring'''







						_A					  =			self.__dict__.copy()
						_A					  =			None
						_A					  =			self.sp_model.serialized_model_proto()
						return state







		def __setstate__(					self      :       Optional[Any]    ,   a__      :       Dict		)       ->						Dict:





						'''simple docstring'''







						_A					  =			d

						# for backward compatibility
						if not hasattr(self    ,   "sp_model_kwargs"		):
										_A					  =			{}

						_A					  =			spm.SentencePieceProcessor(**self.sp_model_kwargs		)
						self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)







		def  a_ (					self      :       str    ,   a__      :       List[int]    ,   a__      :       Optional[List[int]] = None		)       ->						List[int]:





						'''simple docstring'''







						if token_ids_a is None:
										return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						_A					  =			[self.cls_token_id]
						_A					  =			[self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep







		def  a_ (					self      :       List[str]    ,   a__      :       List[int]    ,   a__      :       Optional[List[int]] = None    ,   a__      :       bool = False		)       ->						List[int]:





						'''simple docstring'''







						if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=a__    ,   token_ids_a=a__    ,   already_has_special_tokens=a__		)

						if token_ids_a is None:
										return [1] + ([0] * len(a__		)) + [1]
						return [1] + ([0] * len(a__		)) + [1, 1] + ([0] * len(a__		)) + [1]







		def  a_ (					self      :       Optional[int]    ,   a__      :       List[int]    ,   a__      :       Optional[List[int]] = None		)       ->						List[int]:





						'''simple docstring'''







						_A					  =			[self.sep_token_id]
						_A					  =			[self.cls_token_id]

						if token_ids_a is None:
										return len(cls + token_ids_a + sep		) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]







		@property
		def  a_ (					self      :       List[Any]		)       ->						Union[str, Any]:





						'''simple docstring'''







						return len(self.fairseq_ids_to_tokens		)







		def  a_ (					self      :       List[str]		)       ->						Dict:





						'''simple docstring'''







						_A					  =			{self.convert_ids_to_tokens(a__		): i for i in range(self.vocab_size		)}
						vocab.update(self.added_tokens_encoder		)
						return vocab







		def  a_ (					self      :       Optional[Any]    ,   a__      :       str		)       ->						List[str]:





						'''simple docstring'''







						return self.sp_model.encode(a__    ,   out_type=a__		)







		def  a_ (					self      :       Any    ,   a__      :       Dict		)       ->						Union[str, Any]:





						'''simple docstring'''







						if token in self.fairseq_tokens_to_ids:
										return self.fairseq_tokens_to_ids[token]
						else:
										return self.unk_token_id







		def  a_ (					self      :       List[str]    ,   a__      :       Dict		)       ->						List[str]:





						'''simple docstring'''







						return self.fairseq_ids_to_tokens[index]







		def  a_ (					self      :       Tuple    ,   a__      :       Union[str, Any]		)       ->						Optional[Any]:





						'''simple docstring'''







						_A					  =			"".join(a__		).replace(a__    ,   " "		).strip()
						return out_string







		def  a_ (					self      :       Optional[Any]    ,   a__      :       str    ,   a__      :       Optional[str] = None		)       ->						Tuple[str]:





						'''simple docstring'''







						if not os.path.isdir(a__		):
										logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""		)
										return
						_A					  =			os.path.join(
						    a__    ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
						_A					  =			os.path.join(
						    a__    ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["monolingual_vocab_file"]    ,   )

						if os.path.abspath(self.vocab_file		) != os.path.abspath(a__		) and os.path.isfile(self.vocab_file		):
										copyfile(self.vocab_file    ,   a__		)
						elif not os.path.isfile(self.vocab_file		):
										with open(a__    ,   "wb"		) as fi:
														_A					  =			self.sp_model.serialized_model_proto()
														fi.write(a__		)

						if os.path.abspath(self.monolingual_vocab_file		) != os.path.abspath(
						    a__		) and os.path.isfile(self.monolingual_vocab_file		):
										copyfile(self.monolingual_vocab_file    ,   a__		)
						elif not os.path.isfile(self.monolingual_vocab_file		):
										with open(a__    ,   "w"    ,   encoding="utf-8"		) as fp:
														for token in self.fairseq_tokens_to_ids:
																		if token not in self.all_special_tokens:
																						fp.write(F"""{str(a__		)} \n"""		)

						return out_vocab_file, out_monolingual_vocab_file



"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a_       							=    {
    "configuration_upernet": ["UperNetConfig"],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		a_       							=    [
		    "UperNetForSemanticSegmentation",
		    "UperNetPreTrainedModel",
		]


if TYPE_CHECKING:
		from .configuration_upernet import UperNetConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_upernet import UperNetForSemanticSegmentation, UperNetPreTrainedModel


else:
		import sys

		a_       							=    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)