


import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


__UpperCamelCase				:  List[str]      					=							logging.get_logger(__name__)


__UpperCamelCase				:  Tuple      					=							{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__UpperCamelCase				:  int      					=							{
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

__UpperCamelCase				:  List[str]      					=							{
    'allenai/led-base-16384': 1_6384,
}





class  lowerCAmelCase__(    lowercase__						):






    '''simple docstring'''





    A_  :     int			= VOCAB_FILES_NAMES
    A_  :     Any			= PRETRAINED_VOCAB_FILES_MAP
    A_  :     Dict			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    A_  :     List[Any]			= LEDTokenizer
    A_  :     Tuple			= ["""input_ids""", """attention_mask"""]





    def __init__(	self				:   Optional[Any]       ,   __snake_case				:   Optional[Any]=None       ,   __snake_case				:   Dict=None       ,   __snake_case				:   Tuple=None       ,   __snake_case				:   Union[str, Any]="replace"       ,   __snake_case				:   Tuple="<s>"       ,   __snake_case				:   Optional[Any]="</s>"       ,   __snake_case				:   Tuple="</s>"       ,   __snake_case				:   List[str]="<s>"       ,   __snake_case				:   Tuple="<unk>"       ,   __snake_case				:   Dict="<pad>"       ,   __snake_case				:   Dict="<mask>"       ,   __snake_case				:   Any=False       ,   __snake_case				:   Any=True       ,   **__snake_case				:   List[Any]       ,   ):





       '''simple docstring'''
       super().__init__(
           __lowercase       ,   __lowercase       ,   tokenizer_file=__lowercase       ,   errors=__lowercase       ,   bos_token=__lowercase       ,   eos_token=__lowercase       ,   sep_token=__lowercase       ,   cls_token=__lowercase       ,   unk_token=__lowercase       ,   pad_token=__lowercase       ,   mask_token=__lowercase       ,   add_prefix_space=__lowercase       ,   trim_offsets=__lowercase       ,   **__lowercase       ,   )

       UpperCAmelCase_							:      Any						=   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
       if pre_tok_state.get('''add_prefix_space'''       ,   __lowercase	) != add_prefix_space:
          UpperCAmelCase_							:      int						=   getattr(__lowercase       ,   pre_tok_state.pop('''type'''	)	)
          UpperCAmelCase_							:      Tuple						=   add_prefix_space
          UpperCAmelCase_							:      List[Any]						=   pre_tok_class(**__lowercase	)

       UpperCAmelCase_							:      Optional[int]						=   add_prefix_space

       # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
       UpperCAmelCase_							:      Tuple						=   '''post_processor'''
       UpperCAmelCase_							:      str						=   getattr(self.backend_tokenizer       ,   __lowercase       ,   __lowercase	)
       if tokenizer_component_instance:
          UpperCAmelCase_							:      List[Any]						=   json.loads(tokenizer_component_instance.__getstate__()	)

          # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
          if "sep" in state:
             UpperCAmelCase_							:      List[str]						=   tuple(state['''sep''']	)
          if "cls" in state:
             UpperCAmelCase_							:      Tuple						=   tuple(state['''cls''']	)

          UpperCAmelCase_							:      Tuple						=   False

          if state.get('''add_prefix_space'''       ,   __lowercase	) != add_prefix_space:
             UpperCAmelCase_							:      str						=   add_prefix_space
             UpperCAmelCase_							:      Union[str, Any]						=   True

          if state.get('''trim_offsets'''       ,   __lowercase	) != trim_offsets:
             UpperCAmelCase_							:      int						=   trim_offsets
             UpperCAmelCase_							:      Union[str, Any]						=   True

          if changes_to_apply:
             UpperCAmelCase_							:      str						=   getattr(__lowercase       ,   state.pop('''type'''	)	)
             UpperCAmelCase_							:      Optional[Any]						=   component_class(**__lowercase	)
             setattr(self.backend_tokenizer       ,   __lowercase       ,   __lowercase	)





    @property
    # Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
    def 					_lowerCamelCase (	self				:   str	):





       '''simple docstring'''
       if self._mask_token is None:
          if self.verbose:
             logger.error('''Using mask_token, but it is not set yet.'''	)
          return None
       return str(self._mask_token	)





    @mask_token.setter
    def 					_lowerCamelCase (	self				:   Optional[int]       ,   __snake_case				:   Dict	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   AddedToken(__lowercase       ,   lstrip=__lowercase       ,   rstrip=__lowercase	) if isinstance(__lowercase       ,   __lowercase	) else value
       UpperCAmelCase_							:      List[str]						=   value





    def 					_lowerCamelCase (	self				:   Any       ,   *__snake_case				:   List[Any]       ,   **__snake_case				:   Optional[Any]	):





       '''simple docstring'''
       UpperCAmelCase_							:      str						=   kwargs.get('''is_split_into_words'''       ,   __lowercase	)

       if is_split_into_words and not self.add_prefix_space:
          raise ValueError(
              f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
              '''to use it with pretokenized inputs.'''	)

       return super()._batch_encode_plus(*__lowercase       ,   **__lowercase	)





    def 					_lowerCamelCase (	self				:   int       ,   *__snake_case				:   Union[str, Any]       ,   **__snake_case				:   List[str]	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   kwargs.get('''is_split_into_words'''       ,   __lowercase	)

       if is_split_into_words and not self.add_prefix_space:
          raise ValueError(
              f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
              '''to use it with pretokenized inputs.'''	)

       return super()._encode_plus(*__lowercase       ,   **__lowercase	)





    def 					_lowerCamelCase (	self				:   Optional[Any]       ,   __snake_case				:   str       ,   __snake_case				:   Optional[str] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      List[str]						=   self._tokenizer.model.save(__lowercase       ,   name=__lowercase	)
       return tuple(__lowercase	)





    def 					_lowerCamelCase (	self				:   List[str]       ,   __snake_case				:   int       ,   __snake_case				:   Optional[int]=None	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   [self.bos_token_id] + token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
          return output

       return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





    def 					_lowerCamelCase (	self				:   int       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   [self.sep_token_id]
       UpperCAmelCase_							:      Optional[Any]						=   [self.cls_token_id]

       if token_ids_a is None:
          return len(cls + token_ids_a + sep	) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]





    def 					_lowerCamelCase (	self				:   Union[str, Any]       ,   __snake_case				:   Union[Dict[str, EncodedInput], BatchEncoding]       ,   __snake_case				:   Optional[int] = None       ,   __snake_case				:   PaddingStrategy = PaddingStrategy.DO_NOT_PAD       ,   __snake_case				:   Optional[int] = None       ,   __snake_case				:   Optional[bool] = None       ,   ):





       '''simple docstring'''
       UpperCAmelCase_							:      Union[str, Any]						=   super()._pad(
           encoded_inputs=__lowercase       ,   max_length=__lowercase       ,   padding_strategy=__lowercase       ,   pad_to_multiple_of=__lowercase       ,   return_attention_mask=__lowercase       ,   )

       # Load from model defaults
       if return_attention_mask is None:
          UpperCAmelCase_							:      Any						=   '''attention_mask''' in self.model_input_names

       if return_attention_mask and "global_attention_mask" in encoded_inputs:
          UpperCAmelCase_							:      str						=   encoded_inputs[self.model_input_names[0]]
          # `global_attention_mask` need to have the same length as other (sequential) inputs.
          UpperCAmelCase_							:      Union[str, Any]						=   len(encoded_inputs['''global_attention_mask''']	) != len(__lowercase	)

          if needs_to_be_padded:
             UpperCAmelCase_							:      Any						=   len(__lowercase	) - len(encoded_inputs['''global_attention_mask''']	)

             if self.padding_side == "right":
                # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                UpperCAmelCase_							:      Dict						=   (
                    encoded_inputs['''global_attention_mask'''] + [-1] * difference
                )
             elif self.padding_side == "left":
                UpperCAmelCase_							:      Dict						=   [-1] * difference + encoded_inputs[
                    '''global_attention_mask'''
                ]
             else:
                raise ValueError('''Invalid padding strategy:''' + str(self.padding_side	)	)

       return encoded_inputs




# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__UpperCamelCase				:  List[Any]      					=							{
    'configuration_vivit': ['VIVIT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'VivitConfig'],
}
try:
       if not is_vision_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __UpperCamelCase				:  Optional[Any]      					=							['VivitImageProcessor']


try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __UpperCamelCase				:  List[Any]      					=							[
           'VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST',
           'VivitModel',
           'VivitPreTrainedModel',
           'VivitForVideoClassification',
       ]


if TYPE_CHECKING:
       from .configuration_vivit import VIVIT_PRETRAINED_CONFIG_ARCHIVE_MAP, VivitConfig

       try:
              if not is_vision_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .image_processing_vivit import VivitImageProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_vivit import (
                  VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  VivitForVideoClassification,
                  VivitModel,
                  VivitPreTrainedModel,
              )


else:
       import sys

       __UpperCamelCase				:  str      					=							_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)