





from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


_lowerCamelCase			       =      logging.get_logger(__name__)

_lowerCamelCase			       =      {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

_lowerCamelCase			       =      {
    'vocab_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'
    },
    'merges_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'
    },
}

_lowerCamelCase			       =      {'allegro/herbert-base-cased': 514}
_lowerCamelCase			       =      {}







class 			__A       (      lowerCamelCase__  ):




      """simple docstring"""







      UpperCAmelCase__						=      VOCAB_FILES_NAMES
      UpperCAmelCase__						=      PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase__						=      PRETRAINED_INIT_CONFIGURATION
      UpperCAmelCase__						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase__						=      HerbertTokenizer


      def __init__(		self      ,							a__=None      ,							a__=None      ,							a__=None      ,							a__="<s>"      ,							a__="<unk>"      ,							a__="<pad>"      ,							a__="<mask>"      ,							a__="</s>"      ,							**a__      ,							):



             """simple docstring"""





             super().__init__(
                 a__      ,							a__      ,							tokenizer_file=a__      ,							cls_token=a__      ,							unk_token=a__      ,							pad_token=a__      ,							mask_token=a__      ,							sep_token=a__      ,							**a__      ,							)


      def 				__snake_case			(		self      ,							a__      ,							a__ = None):



             """simple docstring"""





             _lowerCamelCase							:     str	=		[self.cls_token_id]
             _lowerCamelCase							:     Any	=		[self.sep_token_id]
             if token_ids_a is None:
                    return cls + token_ids_a + sep

             return cls + token_ids_a + sep + token_ids_a + sep


      def 				__snake_case			(		self      ,							a__      ,							a__ = None      ,							a__ = False):



             """simple docstring"""





             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=a__      ,							token_ids_a=a__      ,							already_has_special_tokens=a__)

             if token_ids_a is None:
                    return [1] + ([0] * len(a__)) + [1]
             return [1] + ([0] * len(a__)) + [1] + ([0] * len(a__)) + [1]


      def 				__snake_case			(		self      ,							a__      ,							a__ = None):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		[self.sep_token_id]
             _lowerCamelCase							:     Dict	=		[self.cls_token_id]

             if token_ids_a is None:
                    return len(cls + token_ids_a + sep) * [0]
             return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



      def 				__snake_case			(		self      ,							a__      ,							a__ = None):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		self._tokenizer.model.save(a__      ,							name=a__)
             return tuple(a__)








import warnings

from ...utils import logging
from .image_processing_donut import DonutImageProcessor


_lowerCamelCase			       =      logging.get_logger(__name__)







class 			__A       (      lowerCamelCase__  ):




      """simple docstring"""







      def __init__(		self      ,							*a__      ,							**a__):



             """simple docstring"""





             warnings.warn(
                 '''The class DonutFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please'''
                 ''' use DonutImageProcessor instead.'''      ,							a__      ,							)
             super().__init__(*a__      ,							**a__)

