





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase			       =      logging.get_logger(__name__)

_lowerCamelCase			       =      '▁'

_lowerCamelCase			       =      {'vocab_file': 'sentencepiece.bpe.model', 'monolingual_vocab_file': 'dict.txt'}

_lowerCamelCase			       =      {
    'vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model',
    },
    'monolingual_vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt',
    },
}

_lowerCamelCase			       =      {'vinai/bartpho-syllable': 1024}







class 			__A       (      lowerCamelCase__  ):




      """simple docstring"""







      UpperCAmelCase__						=      VOCAB_FILES_NAMES
      UpperCAmelCase__						=      PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase__						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase__						=      ["""input_ids""", """attention_mask"""]


      def __init__(		self      ,							a__      ,							a__      ,							a__="<s>"      ,							a__="</s>"      ,							a__="</s>"      ,							a__="<s>"      ,							a__="<unk>"      ,							a__="<pad>"      ,							a__="<mask>"      ,							a__ = None      ,							**a__      ,							):



             """simple docstring"""





             _lowerCamelCase							:     Tuple	=		AddedToken(a__      ,							lstrip=a__      ,							rstrip=a__) if isinstance(a__      ,							a__) else mask_token

             _lowerCamelCase							:     List[str]	=		{} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 bos_token=a__      ,							eos_token=a__      ,							unk_token=a__      ,							sep_token=a__      ,							cls_token=a__      ,							pad_token=a__      ,							mask_token=a__      ,							sp_model_kwargs=self.sp_model_kwargs      ,							**a__      ,							)

             _lowerCamelCase							:     Optional[Any]	=		vocab_file
             _lowerCamelCase							:     Tuple	=		monolingual_vocab_file
             _lowerCamelCase							:     List[Any]	=		spm.SentencePieceProcessor(**self.sp_model_kwargs)
             self.sp_model.Load(str(a__))

             # Load the reduced vocab

             # Keep order of special tokens for backward compatibility
             _lowerCamelCase							:     List[Any]	=		{}
             _lowerCamelCase							:     int	=		0
             for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
                    if str(a__) not in self.fairseq_tokens_to_ids:
                           _lowerCamelCase							:     Any	=		cnt
                           cnt += 1
             with open(a__      ,							'''r'''      ,							encoding='''utf-8''') as f:
                    for line in f.readlines():
                           _lowerCamelCase							:     Union[str, Any]	=		line.strip().split()[0]
                           _lowerCamelCase							:     List[Any]	=		len(self.fairseq_tokens_to_ids)
             if str(a__) not in self.fairseq_tokens_to_ids:
                    _lowerCamelCase							:     int	=		len(self.fairseq_tokens_to_ids)

             _lowerCamelCase							:     Optional[Any]	=		{v: k for k, v in self.fairseq_tokens_to_ids.items()}


      def __getstate__(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		self.__dict__.copy()
             _lowerCamelCase							:     Dict	=		None
             _lowerCamelCase							:     Tuple	=		self.sp_model.serialized_model_proto()
             return state


      def __setstate__(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		d

             # for backward compatibility
             if not hasattr(self      ,							'''sp_model_kwargs'''):
                    _lowerCamelCase							:     List[Any]	=		{}

             _lowerCamelCase							:     Tuple	=		spm.SentencePieceProcessor(**self.sp_model_kwargs)
             self.sp_model.LoadFromSerializedProto(self.sp_model_proto)


      def 				__snake_case			(		self      ,							a__      ,							a__ = None):



             """simple docstring"""





             if token_ids_a is None:
                    return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
             _lowerCamelCase							:     int	=		[self.cls_token_id]
             _lowerCamelCase							:     Dict	=		[self.sep_token_id]
             return cls + token_ids_a + sep + sep + token_ids_a + sep


      def 				__snake_case			(		self      ,							a__      ,							a__ = None      ,							a__ = False):



             """simple docstring"""





             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=a__      ,							token_ids_a=a__      ,							already_has_special_tokens=a__)

             if token_ids_a is None:
                    return [1] + ([0] * len(a__)) + [1]
             return [1] + ([0] * len(a__)) + [1, 1] + ([0] * len(a__)) + [1]


      def 				__snake_case			(		self      ,							a__      ,							a__ = None):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		[self.sep_token_id]
             _lowerCamelCase							:     Tuple	=		[self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep) * [0]
             return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


      @property
      def 				__snake_case			(		self):



             """simple docstring"""





             return len(self.fairseq_ids_to_tokens)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     str	=		{self.convert_ids_to_tokens(a__): i for i in range(self.vocab_size)}
             vocab.update(self.added_tokens_encoder)
             return vocab


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             return self.sp_model.encode(a__      ,							out_type=a__)


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             if token in self.fairseq_tokens_to_ids:
                    return self.fairseq_tokens_to_ids[token]
             else:
                    return self.unk_token_id


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             return self.fairseq_ids_to_tokens[index]


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     List[str]	=		''''''.join(a__).replace(a__      ,							''' ''').strip()
             return out_string



      def 				__snake_case			(		self      ,							a__      ,							a__ = None):



             """simple docstring"""





             if not os.path.isdir(a__):
                    logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
                    return
             _lowerCamelCase							:     Optional[int]	=		os.path.join(
                 a__      ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
             _lowerCamelCase							:     Tuple	=		os.path.join(
                 a__      ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''monolingual_vocab_file''']      ,							)

             if os.path.abspath(self.vocab_file) != os.path.abspath(a__) and os.path.isfile(self.vocab_file):
                    copyfile(self.vocab_file      ,							a__)
             elif not os.path.isfile(self.vocab_file):
                    with open(a__      ,							'''wb''') as fi:
                           _lowerCamelCase							:     List[str]	=		self.sp_model.serialized_model_proto()
                           fi.write(a__)

             if os.path.abspath(self.monolingual_vocab_file) != os.path.abspath(
                 a__) and os.path.isfile(self.monolingual_vocab_file):
                    copyfile(self.monolingual_vocab_file      ,							a__)
             elif not os.path.isfile(self.monolingual_vocab_file):
                    with open(a__      ,							'''w'''      ,							encoding='''utf-8''') as fp:
                           for token in self.fairseq_tokens_to_ids:
                                  if token not in self.all_special_tokens:
                                         fp.write(F"""{str(a__)} \n""")

             return out_vocab_file, out_monolingual_vocab_file








import warnings

from ...utils import logging
from .image_processing_donut import DonutImageProcessor


_lowerCamelCase			       =      logging.get_logger(__name__)







class 			__A       (      lowerCamelCase__  ):




      """simple docstring"""







      def __init__(		self      ,							*a__      ,							**a__):



             """simple docstring"""





             warnings.warn(
                 '''The class DonutFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'''
                 ''' use DonutImageProcessor instead.'''      ,							a__      ,							)
             super().__init__(*a__      ,							**a__)

